diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b4df585a9abd51b142020523fb2239f521b17c1f --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +# YuLan hybrid continued-pretraining checkpoints + +Archive upload is in progress. Do not assume a checkpoint is complete until `UPLOAD_COMPLETE.json` exists. + +- Stage15 final / Stage16a starting checkpoint: `iter_0006855`. +- Stage16a final checkpoint: `iter_0002384`. +- `checkpoints/`: native Megatron Core distributed checkpoints, including optimizer/training state as present in the source. +- `hf/`: the existing Hugging Face exports recovered from the YuLan object store. +- Training logs and launch scripts retain the stage-relative layout. HF export logs are under `hf_export_logs/`. +- Files listed in `archive-manifest.json` are verified against remote sizes and content hashes at completion. + +This repository is public at the owner's request. Original training checkpoint files are read in place through task-owned hardlinks. diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1440.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1440.log new file mode 100644 index 0000000000000000000000000000000000000000..c3333d07c65bc109f85e3675307e00e0985cdb83 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1440.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_1440 ++ mkdir -p /tmp/mcore2hf_daemon_1440 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 1440 ']' ++ MANUAL_ITER=1440 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 ++ CHECKPOINT_BASENAME=load_view1440 ++ [[ load_view1440 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_1440/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_1440/mcore2hf-config.YyPuwk ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_1440/mcore2hf-config.YyPuwk:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_1440/mcore2hf-config.YyPuwk:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 +W1015 08:28:54.562000 776063 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 08:28:54.562000 776063 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 +W1015 08:29:09.547000 776651 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 08:29:09.547000 776651 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_1440/mcore2hf-config.YyPuwk +W1015 08:29:22.443000 776949 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 08:29:22.443000 776949 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_1440/mcore2hf-config.YyPuwk/config.json' +✅config.json: /tmp/mcore2hf_daemon_1440/mcore2hf-config.YyPuwk/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_1440/mcore2hf-config.YyPuwk/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_1440/mcore2hf-config.YyPuwk ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 1440 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 ++ local ITER_NUM=1440 ++ '[' -z 1440 ']' +++ printf %07d 1440 ++ local ITER_BASENAME=iter_0001440 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440/iter_0001440 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440/iter_0001440 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440/latest_checkpointed_iteration.txt ++ LATEST_VALUE=1440 ++ '[' 1440 = 1440 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=1440 ++ echo -e '\033[32mLatest checkpoint iteration: 1440\033[0m\n' +Latest checkpoint iteration: 1440 + ++ I=1440 ++ [[ 1440 =~ ^[0-9]+$ ]] +++ printf %07d 1440 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440/iter_0001440 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440/iter_0001440/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440/iter_1440-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1440-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1440-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1440-hf true false bf16 /tmp/mcore2hf_daemon_1440/mcore2hf-config.YyPuwk 1440 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1440-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 45415 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_1440/mcore2hf-config.YyPuwk --hf-dir /tmp/mcore2hf_daemon_1440/mcore2hf-config.YyPuwk --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1440-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 1440 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1015 08:29:42.216000 777370 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 08:29:42.216000 777370 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 1440 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_1440/mcore2hf-config.YyPuwk + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1440-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_1440/mcore2hf-config.YyPuwk + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.132 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_YyPuwk/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_YyPuwk/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_YyPuwk.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 at iteration 1440 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1440 [ t 1/1, p 1/1 ] at iteration 1440 +[convert] synchronizer created in 388.13s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.000484466552734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10205078125 mean=0.001495361328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07861328125 mean=-0.00012969970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10498046875 mean=0.00011396408081054688 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08447265625 mean=-0.00021266937255859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0791015625 mean=0.000873565673828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=-0.000255584716796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07666015625 mean=0.00150299072265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09619140625 mean=-0.00021648406982421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0732421875 mean=-0.000774383544921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09130859375 mean=-8.106231689453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06201171875 mean=-0.000240325927734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09326171875 mean=-9.250640869140625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.062255859375 mean=7.963180541992188e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=3.62396240234375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.055908203125 mean=-0.000339508056640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=5.555152893066406e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.053955078125 mean=-0.00031280517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08203125 mean=-6.794929504394531e-06 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0654296875 mean=-0.000469207763671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.0001983642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0537109375 mean=-0.0005035400390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=8.58306884765625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=-0.00113677978515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.0001373291015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04638671875 mean=-0.0009307861328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.000217437744140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048583984375 mean=-0.000438690185546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.0002803802490234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046630859375 mean=-0.00054168701171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.000316619873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.045654296875 mean=-0.00167083740234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.0002574920654296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046142578125 mean=-0.001983642578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.000545501708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048828125 mean=-0.001678466796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.0003376007080078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0498046875 mean=-0.00122833251953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.0003490447998046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04052734375 mean=-0.002227783203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=0.00028228759765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=-0.0028533935546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053466796875 mean=0.0004177093505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03662109375 mean=-0.00213623046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.0004177093505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03466796875 mean=-0.00250244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.00037384033203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032958984375 mean=-0.0016326904296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.00029754638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0306396484375 mean=-0.00154876708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0003910064697265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0299072265625 mean=-0.00174713134765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047607421875 mean=0.0003719329833984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026611328125 mean=-0.000789642333984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.000507354736328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.000804901123046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.00080108642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0247802734375 mean=-0.00064849853515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.000530242919921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0260009765625 mean=-0.0004825592041015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046142578125 mean=0.00012969970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0250244140625 mean=-0.000301361083984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=7.772445678710938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0274658203125 mean=9.822845458984375e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.00013828277587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026611328125 mean=0.0001316070556640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05712890625 mean=3.600120544433594e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0322265625 mean=0.0003643035888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05908203125 mean=3.314018249511719e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=0.0009918212890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05859375 mean=8.20159912109375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=0.00138092041015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=9.775161743164062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=0.00165557861328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=6.0558319091796875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03662109375 mean=0.001373291015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=8.916854858398438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025634765625 mean=0.00051116943359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=8.916854858398438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=0.00021457672119140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.000308990478515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0206298828125 mean=-0.0004138946533203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=0.00015354156494140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0198974609375 mean=-0.00013828277587890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035400390625 mean=0.0004329681396484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021484375 mean=-0.00038909912109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=9.1552734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0234375 mean=-0.000865936279296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.00028228759765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0206298828125 mean=-0.00054931640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.0001773834228515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023681640625 mean=-0.000850677490234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-8.678436279296875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019775390625 mean=-0.000514984130859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.033935546875 mean=-0.00015163421630859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0213623046875 mean=-0.00103759765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-0.00012493133544921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.000797271728515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=-1.6927719116210938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.0010223388671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0400390625 mean=0.00018978118896484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0306396484375 mean=-0.00122833251953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=9.441375732421875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05322265625 mean=-0.002532958984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=7.62939453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041015625 mean=-0.00185394287109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04296875 mean=-0.00013637542724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=0.0025634765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.94s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1440-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 78.78s +Conversion finished in 469.2949695587158 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1440-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1440-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_1440/mcore2hf-config.YyPuwk ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1680.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1680.log new file mode 100644 index 0000000000000000000000000000000000000000..2305bff19ec89820c5a9a719c26002b265a89efa --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter1680.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_1680 ++ mkdir -p /tmp/mcore2hf_daemon_1680 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 1680 ']' ++ MANUAL_ITER=1680 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 ++ CHECKPOINT_BASENAME=load_view1680 ++ [[ load_view1680 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_1680/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_1680/mcore2hf-config.p1xir8 ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_1680/mcore2hf-config.p1xir8:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_1680/mcore2hf-config.p1xir8:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 +W1015 23:33:37.395000 918545 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 23:33:37.395000 918545 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 +W1015 23:33:52.741000 918936 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 23:33:52.741000 918936 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_1680/mcore2hf-config.p1xir8 +W1015 23:34:05.893000 919275 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 23:34:05.893000 919275 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_1680/mcore2hf-config.p1xir8/config.json' +✅config.json: /tmp/mcore2hf_daemon_1680/mcore2hf-config.p1xir8/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_1680/mcore2hf-config.p1xir8/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_1680/mcore2hf-config.p1xir8 ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 1680 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 ++ local ITER_NUM=1680 ++ '[' -z 1680 ']' +++ printf %07d 1680 ++ local ITER_BASENAME=iter_0001680 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680/iter_0001680 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680/iter_0001680 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680/latest_checkpointed_iteration.txt ++ LATEST_VALUE=1680 ++ '[' 1680 = 1680 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=1680 ++ echo -e '\033[32mLatest checkpoint iteration: 1680\033[0m\n' +Latest checkpoint iteration: 1680 + ++ I=1680 ++ [[ 1680 =~ ^[0-9]+$ ]] +++ printf %07d 1680 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680/iter_0001680 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680/iter_0001680/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680/iter_1680-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1680-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1680-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1680-hf true false bf16 /tmp/mcore2hf_daemon_1680/mcore2hf-config.p1xir8 1680 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1680-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 51193 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_1680/mcore2hf-config.p1xir8 --hf-dir /tmp/mcore2hf_daemon_1680/mcore2hf-config.p1xir8 --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1680-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 1680 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1015 23:34:26.386000 919813 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1015 23:34:26.386000 919813 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 1680 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_1680/mcore2hf-config.p1xir8 + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1680-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_1680/mcore2hf-config.p1xir8 + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.133 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_p1xir8/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_p1xir8/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_p1xir8.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 at iteration 1680 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view1680 [ t 1/1, p 1/1 ] at iteration 1680 +[convert] synchronizer created in 391.42s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.0004749298095703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10107421875 mean=0.0014495849609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07861328125 mean=-0.00012874603271484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10400390625 mean=6.389617919921875e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08447265625 mean=-0.0002117156982421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07861328125 mean=0.00106048583984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=-0.0002613067626953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0751953125 mean=0.0015106201171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-0.0002193450927734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.072265625 mean=-0.000728607177734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0908203125 mean=-9.059906005859375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0615234375 mean=-0.0003261566162109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09326171875 mean=-9.775161743164062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.061279296875 mean=4.363059997558594e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=3.9577484130859375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05517578125 mean=-0.0003509521484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=5.435943603515625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05322265625 mean=-0.000335693359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08203125 mean=-1.138448715209961e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06494140625 mean=-0.000713348388671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.0001964569091796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05322265625 mean=-0.0002994537353515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=7.677078247070312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=-0.00104522705078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00012969970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04541015625 mean=-0.000797271728515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.00021648406982421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0478515625 mean=-0.0004138946533203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.0002899169921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0458984375 mean=-0.00052642822265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.000316619873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044677734375 mean=-0.0013885498046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.00026702880859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044677734375 mean=-0.00182342529296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.00054931640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04833984375 mean=-0.00156402587890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.000347137451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04931640625 mean=-0.00113677978515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.0003414154052734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03955078125 mean=-0.0021820068359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055908203125 mean=0.00028228759765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=-0.00286865234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.00041961669921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.035888671875 mean=-0.0020904541015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.0004177093505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.033935546875 mean=-0.00225830078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.0003814697265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0322265625 mean=-0.00141143798828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.000301361083984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02978515625 mean=-0.00144195556640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.000392913818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0291748046875 mean=-0.0015411376953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047607421875 mean=0.00037384033203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0263671875 mean=-0.00060272216796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.00051116943359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0234375 mean=-0.0005950927734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0008087158203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0245361328125 mean=-0.000865936279296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.00054168701171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025634765625 mean=-0.000461578369140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04638671875 mean=0.0001392364501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02490234375 mean=-0.0001392364501953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=7.82012939453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0272216796875 mean=9.298324584960938e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.000133514404296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0262451171875 mean=0.00013828277587890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056884765625 mean=4.1484832763671875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03173828125 mean=0.0003509521484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=3.123283386230469e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=0.000972747802734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05859375 mean=7.963180541992188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=0.001434326171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=9.822845458984375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=0.00164794921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=5.7220458984375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03662109375 mean=0.00139617919921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=8.7738037109375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0252685546875 mean=0.0004749298095703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=8.726119995117188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0234375 mean=0.0002803802490234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.000301361083984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020263671875 mean=-0.0003452301025390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=0.0001583099365234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0194091796875 mean=-8.58306884765625e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=0.0004405975341796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021240234375 mean=-0.00038909912109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=9.393692016601562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0230712890625 mean=-0.00072479248046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.0002536773681640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0203857421875 mean=-0.0005645751953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.0001888275146484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0233154296875 mean=-0.000850677490234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-7.2479248046875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.01953125 mean=-0.0004444122314453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.00014019012451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020751953125 mean=-0.0009918212890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-0.00013637542724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.000690460205078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=-2.47955322265625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.001068115234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0400390625 mean=0.00018310546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0302734375 mean=-0.00140380859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=8.96453857421875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052978515625 mean=-0.0024261474609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=7.43865966796875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.040771484375 mean=-0.0019683837890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04296875 mean=-0.0001392364501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=0.0025634765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.94s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1680-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 75.68s +Conversion finished in 469.435396194458 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1680-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_1680-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_1680/mcore2hf-config.p1xir8 ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2080.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2080.log new file mode 100644 index 0000000000000000000000000000000000000000..9973f957a40a84e97b92aed1adb1f849495c7d80 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2080.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_2080 ++ mkdir -p /tmp/mcore2hf_daemon_2080 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 2080 ']' ++ MANUAL_ITER=2080 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 ++ CHECKPOINT_BASENAME=load_view2080 ++ [[ load_view2080 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_2080/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_2080/mcore2hf-config.64dNfH ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_2080/mcore2hf-config.64dNfH:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_2080/mcore2hf-config.64dNfH:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 +W1017 00:41:52.517000 649383 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 00:41:52.517000 649383 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 +W1017 00:42:05.256000 649804 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 00:42:05.256000 649804 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_2080/mcore2hf-config.64dNfH +W1017 00:42:18.469000 650008 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 00:42:18.469000 650008 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_2080/mcore2hf-config.64dNfH/config.json' +✅config.json: /tmp/mcore2hf_daemon_2080/mcore2hf-config.64dNfH/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_2080/mcore2hf-config.64dNfH/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_2080/mcore2hf-config.64dNfH ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 2080 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 ++ local ITER_NUM=2080 ++ '[' -z 2080 ']' +++ printf %07d 2080 ++ local ITER_BASENAME=iter_0002080 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080/iter_0002080 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080/iter_0002080 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080/latest_checkpointed_iteration.txt ++ LATEST_VALUE=2080 ++ '[' 2080 = 2080 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=2080 ++ echo -e '\033[32mLatest checkpoint iteration: 2080\033[0m\n' +Latest checkpoint iteration: 2080 + ++ I=2080 ++ [[ 2080 =~ ^[0-9]+$ ]] +++ printf %07d 2080 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080/iter_0002080 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080/iter_0002080/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080/iter_2080-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2080-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2080-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2080-hf true false bf16 /tmp/mcore2hf_daemon_2080/mcore2hf-config.64dNfH 2080 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2080-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 42419 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_2080/mcore2hf-config.64dNfH --hf-dir /tmp/mcore2hf_daemon_2080/mcore2hf-config.64dNfH --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2080-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 2080 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1017 00:42:38.440000 650528 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 00:42:38.440000 650528 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 2080 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_2080/mcore2hf-config.64dNfH + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2080-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_2080/mcore2hf-config.64dNfH + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.130 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_64dNfH/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_64dNfH/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_64dNfH.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 at iteration 2080 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2080 [ t 1/1, p 1/1 ] at iteration 2080 +[convert] synchronizer created in 387.31s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.000476837158203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10009765625 mean=0.0015106201171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.078125 mean=-0.000118255615234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10302734375 mean=0.0001659393310546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=-0.0001983642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0771484375 mean=0.0007476806640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08740234375 mean=-0.000278472900390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07373046875 mean=0.001434326171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-0.000225067138671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0712890625 mean=-0.0006561279296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0908203125 mean=-0.0001087188720703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.060546875 mean=-0.0002994537353515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0927734375 mean=-0.00011205673217773438 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06005859375 mean=0.00013065338134765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08740234375 mean=4.9114227294921875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05419921875 mean=-0.000492095947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=5.745887756347656e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05224609375 mean=-0.00032806396484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08203125 mean=-2.5272369384765625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=-0.0007171630859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.0001983642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0517578125 mean=-0.000308990478515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=7.009506225585938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04150390625 mean=-0.00086212158203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00013256072998046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04443359375 mean=-0.00081634521484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.00022029876708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046875 mean=-0.000431060791015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.000286102294921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04443359375 mean=-0.0003986358642578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.0003261566162109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=-0.00147247314453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06640625 mean=0.00026702880859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=-0.00176239013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0693359375 mean=0.000560760498046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04736328125 mean=-0.0015716552734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.0003528594970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0478515625 mean=-0.0011138916015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060791015625 mean=0.000347137451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.038330078125 mean=-0.0020904541015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055908203125 mean=0.0002803802490234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041015625 mean=-0.0028076171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.000415802001953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03466796875 mean=-0.0021209716796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.0004138946533203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03271484375 mean=-0.0022430419921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.000377655029296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0311279296875 mean=-0.0015411376953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.0003032684326171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02880859375 mean=-0.0012359619140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0003795623779296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0284423828125 mean=-0.00127410888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0478515625 mean=0.0003681182861328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025390625 mean=-0.0006256103515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048583984375 mean=0.0005035400390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02294921875 mean=-0.00064849853515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=0.000812530517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.000774383544921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.0005340576171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02490234375 mean=-0.0003643035888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04638671875 mean=0.00013637542724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0245361328125 mean=-2.8371810913085938e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=7.390975952148438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0267333984375 mean=8.726119995117188e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02587890625 mean=0.0003376007080078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056884765625 mean=3.981590270996094e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03125 mean=0.000507354736328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=3.552436828613281e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0419921875 mean=0.0011138916015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058349609375 mean=8.106231689453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=0.00145721435546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=9.965896606445312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036376953125 mean=0.0016326904296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055419921875 mean=5.7220458984375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036376953125 mean=0.00148773193359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=9.059906005859375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0250244140625 mean=0.000644683837890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048583984375 mean=8.916854858398438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0233154296875 mean=0.0003032684326171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.043701171875 mean=0.0002994537353515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019775390625 mean=-0.00031280517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=0.0001506805419921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0189208984375 mean=-0.00016880035400390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=0.00043487548828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0206298828125 mean=-0.000244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035400390625 mean=8.869171142578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0225830078125 mean=-0.000751495361328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.0002536773681640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0198974609375 mean=-0.0004558563232421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.000186920166015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.022705078125 mean=-0.000652313232421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-7.295608520507812e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019287109375 mean=-0.000522613525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.000152587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020263671875 mean=-0.00084686279296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037841796875 mean=-0.00013637542724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023193359375 mean=-0.0006561279296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=-2.47955322265625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0233154296875 mean=-0.0012359619140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=0.00018405914306640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0296630859375 mean=-0.00146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=0.00011157989501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052490234375 mean=-0.002349853515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=6.67572021484375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0400390625 mean=-0.0021514892578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.042724609375 mean=-0.00014972686767578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.002593994140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.94s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2080-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 81.47s +Conversion finished in 471.3128161430359 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2080-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2080-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_2080/mcore2hf-config.64dNfH ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2160.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2160.log new file mode 100644 index 0000000000000000000000000000000000000000..d7827abf1e6b9fd0fda6c3919c480ef952136d3f --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter2160.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_2160 ++ mkdir -p /tmp/mcore2hf_daemon_2160 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 2160 ']' ++ MANUAL_ITER=2160 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 ++ CHECKPOINT_BASENAME=load_view2160 ++ [[ load_view2160 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_2160/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_2160/mcore2hf-config.Hqkj3y ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_2160/mcore2hf-config.Hqkj3y:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_2160/mcore2hf-config.Hqkj3y:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 +W1017 05:41:15.248000 973824 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 05:41:15.248000 973824 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 +W1017 05:41:29.826000 974285 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 05:41:29.826000 974285 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_2160/mcore2hf-config.Hqkj3y +W1017 05:41:42.694000 974609 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 05:41:42.694000 974609 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_2160/mcore2hf-config.Hqkj3y/config.json' +✅config.json: /tmp/mcore2hf_daemon_2160/mcore2hf-config.Hqkj3y/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_2160/mcore2hf-config.Hqkj3y/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_2160/mcore2hf-config.Hqkj3y ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 2160 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 ++ local ITER_NUM=2160 ++ '[' -z 2160 ']' +++ printf %07d 2160 ++ local ITER_BASENAME=iter_0002160 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160/iter_0002160 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160/iter_0002160 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160/latest_checkpointed_iteration.txt ++ LATEST_VALUE=2160 ++ '[' 2160 = 2160 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=2160 ++ echo -e '\033[32mLatest checkpoint iteration: 2160\033[0m\n' +Latest checkpoint iteration: 2160 + ++ I=2160 ++ [[ 2160 =~ ^[0-9]+$ ]] +++ printf %07d 2160 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160/iter_0002160 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160/iter_0002160/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160/iter_2160-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2160-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2160-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2160-hf true false bf16 /tmp/mcore2hf_daemon_2160/mcore2hf-config.Hqkj3y 2160 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2160-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 51909 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_2160/mcore2hf-config.Hqkj3y --hf-dir /tmp/mcore2hf_daemon_2160/mcore2hf-config.Hqkj3y --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2160-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 2160 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1017 05:42:02.930000 975035 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1017 05:42:02.930000 975035 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 2160 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_2160/mcore2hf-config.Hqkj3y + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2160-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_2160/mcore2hf-config.Hqkj3y + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.131 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_Hqkj3y/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_Hqkj3y/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_Hqkj3y.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 at iteration 2160 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view2160 [ t 1/1, p 1/1 ] at iteration 2160 +[convert] synchronizer created in 386.25s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.000476837158203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10009765625 mean=0.0015716552734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.078125 mean=-0.00012063980102539062 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1025390625 mean=0.0002956390380859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=-0.00020313262939453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0771484375 mean=0.000652313232421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08740234375 mean=-0.0002841949462890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07373046875 mean=0.00131988525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-0.00022602081298828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0712890625 mean=-0.000545501708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0908203125 mean=-0.00011110305786132812 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.060791015625 mean=-0.000339508056640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0927734375 mean=-0.00011730194091796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0595703125 mean=0.0001220703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08740234375 mean=5.054473876953125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054443359375 mean=-0.0004367828369140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=5.745887756347656e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052490234375 mean=-0.0003070831298828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08203125 mean=-2.765655517578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06298828125 mean=-0.000713348388671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00019550323486328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.051513671875 mean=-0.000316619873046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=6.4849853515625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04150390625 mean=-0.000820159912109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00013065338134765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044189453125 mean=-0.00089263916015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07421875 mean=0.0002155303955078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046630859375 mean=-0.000400543212890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.000286102294921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04443359375 mean=-0.0004482269287109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.000324249267578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=-0.00136566162109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06640625 mean=0.0002651214599609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=-0.001739501953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0693359375 mean=0.000560760498046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047119140625 mean=-0.001556396484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.000354766845703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047607421875 mean=-0.00103759765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060791015625 mean=0.0003490447998046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0380859375 mean=-0.001953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05615234375 mean=0.00028228759765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.040771484375 mean=-0.00286865234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.0004177093505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.034423828125 mean=-0.0019683837890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.0004177093505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032470703125 mean=-0.0022735595703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050048828125 mean=0.0003757476806640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031005859375 mean=-0.00150299072265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.0003032684326171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.028564453125 mean=-0.00124359130859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0003719329833984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0283203125 mean=-0.00138092041015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0478515625 mean=0.0003681182861328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025146484375 mean=-0.000667572021484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.0005035400390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02294921875 mean=-0.000637054443359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=0.000812530517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0235595703125 mean=-0.000736236572265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.0005340576171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0247802734375 mean=-0.0003509521484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046630859375 mean=0.00013828277587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0242919921875 mean=-0.000152587890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=7.677078247070312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026611328125 mean=0.000152587890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.00013065338134765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0257568359375 mean=0.00020122528076171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056884765625 mean=3.933906555175781e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0311279296875 mean=0.000457763671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=3.790855407714844e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041748046875 mean=0.00107574462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058349609375 mean=7.82012939453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=0.00148773193359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=0.00010251998901367188 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036376953125 mean=0.00160980224609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055419921875 mean=5.412101745605469e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036376953125 mean=0.00145721435546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=8.821487426757812e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02490234375 mean=0.00061798095703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=8.96453857421875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0233154296875 mean=0.0004425048828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.00029754638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.01953125 mean=-0.000331878662109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=0.00014781951904296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.018798828125 mean=-0.00017261505126953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=0.000438690185546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0206298828125 mean=-0.000278472900390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=8.916854858398438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0224609375 mean=-0.000762939453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.0002574920654296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019775390625 mean=-0.0003833770751953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.00018405914306640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.022705078125 mean=-0.000823974609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03369140625 mean=-7.05718994140625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0191650390625 mean=-0.00046539306640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.000156402587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0201416015625 mean=-0.00101470947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037841796875 mean=-0.0001430511474609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023193359375 mean=-0.00079345703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=-2.2530555725097656e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0234375 mean=-0.0012359619140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=0.0001773834228515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0296630859375 mean=-0.001312255859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=0.00011014938354492188 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052490234375 mean=-0.0023345947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=6.532669067382812e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0400390625 mean=-0.0022735595703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04296875 mean=-0.0001468658447265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=0.0025482177734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.95s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2160-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 76.03s +Conversion finished in 464.6727967262268 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2160-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_2160-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_2160/mcore2hf-config.Hqkj3y ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter240.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter240.log new file mode 100644 index 0000000000000000000000000000000000000000..fd47915193e9f5e6bd2afbac1521478eaad44bd5 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter240.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_240 ++ mkdir -p /tmp/mcore2hf_daemon_240 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 240 ']' ++ MANUAL_ITER=240 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 ++ CHECKPOINT_BASENAME=load_view240 ++ [[ load_view240 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_240/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_240/mcore2hf-config.1wqFVC ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_240/mcore2hf-config.1wqFVC:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_240/mcore2hf-config.1wqFVC:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 +W1012 04:58:56.945000 579777 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1012 04:58:56.945000 579777 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 +W1012 04:59:11.359000 580161 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1012 04:59:11.359000 580161 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_240/mcore2hf-config.1wqFVC +W1012 04:59:23.745000 580456 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1012 04:59:23.745000 580456 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_240/mcore2hf-config.1wqFVC/config.json' +✅config.json: /tmp/mcore2hf_daemon_240/mcore2hf-config.1wqFVC/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_240/mcore2hf-config.1wqFVC/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_240/mcore2hf-config.1wqFVC ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 240 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 ++ local ITER_NUM=240 ++ '[' -z 240 ']' +++ printf %07d 240 ++ local ITER_BASENAME=iter_0000240 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240/iter_0000240 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240/iter_0000240 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240/latest_checkpointed_iteration.txt ++ LATEST_VALUE=240 ++ '[' 240 = 240 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=240 ++ echo -e '\033[32mLatest checkpoint iteration: 240\033[0m\n' +Latest checkpoint iteration: 240 + ++ I=240 ++ [[ 240 =~ ^[0-9]+$ ]] +++ printf %07d 240 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240/iter_0000240 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240/iter_0000240/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240/iter_240-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_240-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_240-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_240-hf true false bf16 /tmp/mcore2hf_daemon_240/mcore2hf-config.1wqFVC 240 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_240-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 55587 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_240/mcore2hf-config.1wqFVC --hf-dir /tmp/mcore2hf_daemon_240/mcore2hf-config.1wqFVC --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_240-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 240 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1012 04:59:43.564000 580954 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1012 04:59:43.564000 580954 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 240 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_240/mcore2hf-config.1wqFVC + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_240-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_240/mcore2hf-config.1wqFVC + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.137 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_1wqFVC/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_1wqFVC/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_1wqFVC.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 at iteration 240 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view240 [ t 1/1, p 1/1 ] at iteration 240 +[convert] synchronizer created in 384.59s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07763671875 mean=-0.000514984130859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1044921875 mean=0.001800537109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.080078125 mean=-0.00015735626220703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1083984375 mean=1.5974044799804688e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08642578125 mean=-0.0002651214599609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08056640625 mean=0.00086212158203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08984375 mean=-0.00021648406982421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.080078125 mean=0.001373291015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09765625 mean=-0.00023937225341796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07470703125 mean=-0.0009002685546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0927734375 mean=-7.486343383789062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=-0.00018215179443359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09521484375 mean=-7.390975952148438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=1.8253922462463379e-06 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08935546875 mean=3.4332275390625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.056396484375 mean=-0.0004100799560546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0859375 mean=3.123283386230469e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054443359375 mean=-0.0003414154052734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08349609375 mean=7.677078247070312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06640625 mean=-0.000446319580078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07763671875 mean=0.0002231597900390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05419921875 mean=-0.00058746337890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0732421875 mean=0.000148773193359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=-0.00099945068359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=0.00016498565673828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046875 mean=-0.00092315673828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07470703125 mean=0.00023651123046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0498046875 mean=-0.000949859619140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0732421875 mean=0.000301361083984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048583984375 mean=-0.0009307861328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07080078125 mean=0.0003070831298828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047607421875 mean=-0.0013580322265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06689453125 mean=0.00028228759765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048095703125 mean=-0.002227783203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.000614166259765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050048828125 mean=-0.001617431640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.068359375 mean=0.0003719329833984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050537109375 mean=-0.00116729736328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.061279296875 mean=0.0004100799560546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0419921875 mean=-0.0023345947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056396484375 mean=0.0003261566162109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04443359375 mean=-0.003021240234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.054443359375 mean=0.0004634857177734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037841796875 mean=-0.002227783203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05322265625 mean=0.0004367828369140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.035888671875 mean=-0.0026397705078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05029296875 mean=0.0003986358642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.034423828125 mean=-0.00179290771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.000324249267578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03173828125 mean=-0.00188446044921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.0004482269287109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.030517578125 mean=-0.001678466796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048583984375 mean=0.00040435791015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027587890625 mean=-0.00104522705078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.00054168701171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.00101470947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.000850677490234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025634765625 mean=-0.000926971435546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.000530242919921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0262451171875 mean=-0.000614166259765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047119140625 mean=0.00013637542724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0255126953125 mean=-0.000286102294921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05078125 mean=9.393692016601562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02783203125 mean=2.9802322387695312e-06 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052734375 mean=0.00018024444580078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027099609375 mean=2.47955322265625e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058349609375 mean=6.4849853515625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0322265625 mean=0.0002498626708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060302734375 mean=5.888938903808594e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=0.0010223388671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06005859375 mean=0.00011110305786132812 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.00135040283203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05859375 mean=0.0001201629638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=0.00148773193359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05712890625 mean=7.677078247070312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=0.00133514404296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053466796875 mean=0.00012969970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02587890625 mean=0.0002288818359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05029296875 mean=0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024169921875 mean=4.982948303222656e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.044921875 mean=0.00035858154296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0208740234375 mean=-0.000476837158203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039306640625 mean=0.00019359588623046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0203857421875 mean=-0.000431060791015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.00042724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0216064453125 mean=-0.00040435791015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036865234375 mean=0.00011920928955078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023681640625 mean=-0.000888824462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=0.0003414154052734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020751953125 mean=-0.0006256103515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037353515625 mean=0.000202178955078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0234375 mean=-0.00080108642578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03466796875 mean=-7.009506225585938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0196533203125 mean=-0.000553131103515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.034423828125 mean=-9.918212890625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0216064453125 mean=-0.00112152099609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-8.96453857421875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023193359375 mean=-0.00091552734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03857421875 mean=4.38690185546875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0233154296875 mean=-0.00112152099609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04052734375 mean=0.0002193450927734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02978515625 mean=-0.001251220703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=9.775161743164062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05126953125 mean=-0.0023345947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03955078125 mean=6.771087646484375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0390625 mean=-0.0019989013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.044189453125 mean=-0.00014400482177734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=0.00262451171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.94s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_240-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 74.53s +Conversion finished in 461.52222418785095 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_240-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_240-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_240/mcore2hf-config.1wqFVC ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter320.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter320.log new file mode 100644 index 0000000000000000000000000000000000000000..5b7dae2ad500523831769f053952ce1c33bafaf3 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter320.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_320 ++ mkdir -p /tmp/mcore2hf_daemon_320 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 320 ']' ++ MANUAL_ITER=320 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 ++ CHECKPOINT_BASENAME=load_view320 ++ [[ load_view320 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_320/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_320/mcore2hf-config.aSCHHD ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_320/mcore2hf-config.aSCHHD:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_320/mcore2hf-config.aSCHHD:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 +W1012 10:02:16.248000 909057 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1012 10:02:16.248000 909057 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 +W1012 10:02:31.567000 909504 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1012 10:02:31.567000 909504 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_320/mcore2hf-config.aSCHHD +W1012 10:02:44.345000 910008 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1012 10:02:44.345000 910008 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_320/mcore2hf-config.aSCHHD/config.json' +✅config.json: /tmp/mcore2hf_daemon_320/mcore2hf-config.aSCHHD/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_320/mcore2hf-config.aSCHHD/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_320/mcore2hf-config.aSCHHD ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 320 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 ++ local ITER_NUM=320 ++ '[' -z 320 ']' +++ printf %07d 320 ++ local ITER_BASENAME=iter_0000320 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320/iter_0000320 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320/iter_0000320 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320/latest_checkpointed_iteration.txt ++ LATEST_VALUE=320 ++ '[' 320 = 320 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=320 ++ echo -e '\033[32mLatest checkpoint iteration: 320\033[0m\n' +Latest checkpoint iteration: 320 + ++ I=320 ++ [[ 320 =~ ^[0-9]+$ ]] +++ printf %07d 320 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320/iter_0000320 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320/iter_0000320/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320/iter_320-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_320-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_320-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_320-hf true false bf16 /tmp/mcore2hf_daemon_320/mcore2hf-config.aSCHHD 320 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_320-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 43705 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_320/mcore2hf-config.aSCHHD --hf-dir /tmp/mcore2hf_daemon_320/mcore2hf-config.aSCHHD --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_320-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 320 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1012 10:03:04.771000 910665 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1012 10:03:04.771000 910665 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 320 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_320/mcore2hf-config.aSCHHD + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_320-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_320/mcore2hf-config.aSCHHD + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.126 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_aSCHHD/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_aSCHHD/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_aSCHHD.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 at iteration 320 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view320 [ t 1/1, p 1/1 ] at iteration 320 +[convert] synchronizer created in 387.19s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07763671875 mean=-0.000522613525390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10400390625 mean=0.00182342529296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.080078125 mean=-0.00015354156494140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10791015625 mean=-2.372264862060547e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08642578125 mean=-0.0002593994140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08056640625 mean=0.000843048095703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08984375 mean=-0.00021839141845703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.080078125 mean=0.00138092041015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09765625 mean=-0.0002422332763671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0751953125 mean=-0.000934600830078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0927734375 mean=-7.963180541992188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=-0.000171661376953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09521484375 mean=-8.0108642578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=1.919269561767578e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08935546875 mean=3.933906555175781e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.056884765625 mean=-0.0004138946533203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08544921875 mean=3.5762786865234375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0546875 mean=-0.000308990478515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08349609375 mean=7.534027099609375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06640625 mean=-0.0004863739013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=0.00021648406982421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054443359375 mean=-0.00046539306640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.0001430511474609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=-0.00110626220703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=0.0001659393310546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047119140625 mean=-0.00084686279296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07470703125 mean=0.00023651123046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0498046875 mean=-0.000896453857421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0732421875 mean=0.0003032684326171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048583984375 mean=-0.000919342041015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07080078125 mean=0.0003070831298828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047607421875 mean=-0.00135040283203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06640625 mean=0.000278472900390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04833984375 mean=-0.0023193359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0693359375 mean=0.00060272216796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050048828125 mean=-0.0016021728515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.068359375 mean=0.000362396240234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05078125 mean=-0.00128173828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06103515625 mean=0.0003986358642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=-0.00238037109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05615234375 mean=0.0003223419189453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04443359375 mean=-0.0029754638671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.054443359375 mean=0.0004634857177734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0380859375 mean=-0.0023040771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052978515625 mean=0.0004329681396484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0361328125 mean=-0.0025787353515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05029296875 mean=0.0003986358642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.034423828125 mean=-0.0018157958984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.0003223419189453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031982421875 mean=-0.00189208984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.00045013427734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03076171875 mean=-0.00167083740234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.000415802001953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027587890625 mean=-0.0010223388671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.000545501708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024169921875 mean=-0.00104522705078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=0.0008544921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02587890625 mean=-0.0009918212890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.0005340576171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026611328125 mean=-0.000614166259765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046875 mean=0.00013637542724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025634765625 mean=-0.00030517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050537109375 mean=9.5367431640625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0279541015625 mean=9.5367431640625e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052734375 mean=0.00018215179443359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02734375 mean=-3.218650817871094e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05810546875 mean=6.389617919921875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032470703125 mean=0.0002918243408203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06005859375 mean=6.103515625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=0.00098419189453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.059814453125 mean=0.00011301040649414062 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.0013580322265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05859375 mean=0.0001220703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037353515625 mean=0.0014801025390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056884765625 mean=8.153915405273438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=0.00119781494140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053466796875 mean=0.00012874603271484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02587890625 mean=0.00025177001953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050048828125 mean=0.0001277923583984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0242919921875 mean=0.00018215179443359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.044921875 mean=0.0003566741943359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02099609375 mean=-0.000514984130859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0390625 mean=0.0001983642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0205078125 mean=-0.0002956390380859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.00042724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02197265625 mean=-0.00035858154296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036865234375 mean=0.00011873245239257812 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.000949859619140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036865234375 mean=0.0003376007080078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0211181640625 mean=-0.00066375732421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=0.00019741058349609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023681640625 mean=-0.000827789306640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.034423828125 mean=-7.2479248046875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019775390625 mean=-0.000576019287109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.034423828125 mean=-0.00010204315185546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0218505859375 mean=-0.00112152099609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-8.630752563476562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0235595703125 mean=-0.000762939453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03857421875 mean=4.363059997558594e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0235595703125 mean=-0.0010833740234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04052734375 mean=0.00021457672119140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0301513671875 mean=-0.0012359619140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=0.00010061264038085938 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0517578125 mean=-0.0023040771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039306640625 mean=7.104873657226562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03955078125 mean=-0.0019683837890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=-0.00014495849609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.002655029296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.97s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_320-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 81.04s +Conversion finished in 470.62169432640076 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_320-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_320-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_320/mcore2hf-config.aSCHHD ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter800.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter800.log new file mode 100644 index 0000000000000000000000000000000000000000..ce11954b99c136c640f447dfafc01012d62735f8 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter800.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_800 ++ mkdir -p /tmp/mcore2hf_daemon_800 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 800 ']' ++ MANUAL_ITER=800 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 ++ CHECKPOINT_BASENAME=load_view800 ++ [[ load_view800 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_800/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_800/mcore2hf-config.m2WLLP ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_800/mcore2hf-config.m2WLLP:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_800/mcore2hf-config.m2WLLP:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 +W1013 16:15:24.600000 19909 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 16:15:24.600000 19909 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 +W1013 16:15:40.084000 20453 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 16:15:40.084000 20453 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_800/mcore2hf-config.m2WLLP +W1013 16:15:53.208000 20764 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 16:15:53.208000 20764 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_800/mcore2hf-config.m2WLLP/config.json' +✅config.json: /tmp/mcore2hf_daemon_800/mcore2hf-config.m2WLLP/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_800/mcore2hf-config.m2WLLP/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_800/mcore2hf-config.m2WLLP ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 800 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 ++ local ITER_NUM=800 ++ '[' -z 800 ']' +++ printf %07d 800 ++ local ITER_BASENAME=iter_0000800 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800/iter_0000800 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800/iter_0000800 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800/latest_checkpointed_iteration.txt ++ LATEST_VALUE=800 ++ '[' 800 = 800 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=800 ++ echo -e '\033[32mLatest checkpoint iteration: 800\033[0m\n' +Latest checkpoint iteration: 800 + ++ I=800 ++ [[ 800 =~ ^[0-9]+$ ]] +++ printf %07d 800 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800/iter_0000800 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800/iter_0000800/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800/iter_800-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_800-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_800-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_800-hf true false bf16 /tmp/mcore2hf_daemon_800/mcore2hf-config.m2WLLP 800 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_800-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 37877 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_800/mcore2hf-config.m2WLLP --hf-dir /tmp/mcore2hf_daemon_800/mcore2hf-config.m2WLLP --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_800-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 800 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1013 16:16:13.178000 21305 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1013 16:16:13.178000 21305 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 800 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_800/mcore2hf-config.m2WLLP + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_800-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_800/mcore2hf-config.m2WLLP + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.137 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_m2WLLP/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_m2WLLP/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_m2WLLP.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 at iteration 800 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view800 [ t 1/1, p 1/1 ] at iteration 800 +[convert] synchronizer created in 382.98s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=-0.000507354736328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10302734375 mean=0.00156402587890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0791015625 mean=-0.00011777877807617188 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10693359375 mean=5.9604644775390625e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08544921875 mean=-0.0002155303955078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08056640625 mean=0.0008392333984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0888671875 mean=-0.00023365020751953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0791015625 mean=0.00154876708984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0966796875 mean=-0.0002193450927734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07470703125 mean=-0.000797271728515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.091796875 mean=-7.724761962890625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=-0.000232696533203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09423828125 mean=-7.486343383789062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=0.00014209747314453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08837890625 mean=3.719329833984375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05712890625 mean=-0.0003604888916015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08447265625 mean=5.793571472167969e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054931640625 mean=-0.00037384033203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08251953125 mean=3.838539123535156e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06640625 mean=-0.0004405975341796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=0.00019931793212890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0546875 mean=-0.000560760498046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=0.0001087188720703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0439453125 mean=-0.000820159912109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00015354156494140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04736328125 mean=-0.00092315673828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07421875 mean=0.0002288818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0498046875 mean=-0.00087738037109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.0002994537353515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048583984375 mean=-0.000560760498046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.00031280517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04736328125 mean=-0.00151824951171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.0002727508544921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0478515625 mean=-0.00213623046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.000568389892578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050048828125 mean=-0.00165557861328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.0003566741943359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.051025390625 mean=-0.00124359130859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.000377655029296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=-0.0023956298828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=0.0002956390380859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04443359375 mean=-0.00311279296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.0004520416259765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.038330078125 mean=-0.0023040771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.0004253387451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.035888671875 mean=-0.00250244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.0003833770751953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.034423828125 mean=-0.00173187255859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.000308990478515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031982421875 mean=-0.00162506103515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0004329681396484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03076171875 mean=-0.00174713134765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0478515625 mean=0.000396728515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0274658203125 mean=-0.0009765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.0005340576171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0244140625 mean=-0.000896453857421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.0008087158203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02587890625 mean=-0.00099945068359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.000530242919921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02685546875 mean=-0.000652313232421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04638671875 mean=0.0001316070556640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025634765625 mean=-0.0003376007080078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=8.726119995117188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.028076171875 mean=-4.935264587402344e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=0.0001659393310546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027099609375 mean=-3.743171691894531e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=5.030632019042969e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032470703125 mean=0.0003681182861328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.059326171875 mean=4.9114227294921875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=0.000804901123046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05908203125 mean=0.00010728836059570312 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.00128173828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057861328125 mean=0.00010919570922851562 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037353515625 mean=0.001556396484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05615234375 mean=7.343292236328125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=0.001190185546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052734375 mean=0.00011444091796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026123046875 mean=0.000339508056640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.00010633468627929688 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0244140625 mean=0.0002593994140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.044189453125 mean=0.0003337860107421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0211181640625 mean=-0.000514984130859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=0.00018310546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0205078125 mean=-0.00020122528076171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.0004291534423828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0218505859375 mean=-0.000335693359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.00010633468627929688 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024169921875 mean=-0.000888824462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.0003070831298828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0211181640625 mean=-0.000514984130859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03662109375 mean=0.00018596649169921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.000926971435546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.033935546875 mean=-7.677078247070312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02001953125 mean=-0.000576019287109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-0.00011920928955078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02197265625 mean=-0.00107574462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=-0.00010156631469726562 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024169921875 mean=-0.00064849853515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03955078125 mean=3.457069396972656e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024169921875 mean=-0.00115203857421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.040283203125 mean=0.00019931793212890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0308837890625 mean=-0.0013275146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03759765625 mean=9.393692016601562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.053466796875 mean=-0.00244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03857421875 mean=9.489059448242188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041259765625 mean=-0.0019073486328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.043212890625 mean=-0.00012874603271484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0439453125 mean=0.0028076171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.94s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_800-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 73.41s +Conversion finished in 458.72578740119934 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_800-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_800-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_800/mcore2hf-config.m2WLLP ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter960.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter960.log new file mode 100644 index 0000000000000000000000000000000000000000..8e398a8891ffb582fef7e909301b9494b87e703c --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/convert_iter960.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/root/.local/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_960 ++ mkdir -p /tmp/mcore2hf_daemon_960 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 960 ']' ++ MANUAL_ITER=960 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 ++ CHECKPOINT_BASENAME=load_view960 ++ [[ load_view960 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_daemon_960/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_daemon_960/mcore2hf-config.jmcWtl ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_daemon_960/mcore2hf-config.jmcWtl:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_daemon_960/mcore2hf-config.jmcWtl:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 +W1014 02:20:04.329000 751090 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 02:20:04.329000 751090 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=524288 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=524288 +++ MAX_POSITION_EMBEDDINGS=524288 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=524288 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 +W1014 02:20:23.685000 752251 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 02:20:23.685000 752251 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 524288 ']' ++ '[' 524288 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_daemon_960/mcore2hf-config.jmcWtl +W1014 02:20:36.100000 752568 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 02:20:36.100000 752568 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_daemon_960/mcore2hf-config.jmcWtl/config.json' +✅config.json: /tmp/mcore2hf_daemon_960/mcore2hf-config.jmcWtl/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_daemon_960/mcore2hf-config.jmcWtl/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 524288, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_daemon_960/mcore2hf-config.jmcWtl ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 960 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 ++ local ITER_NUM=960 ++ '[' -z 960 ']' +++ printf %07d 960 ++ local ITER_BASENAME=iter_0000960 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960/iter_0000960 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960/iter_0000960 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960/latest_checkpointed_iteration.txt ++ LATEST_VALUE=960 ++ '[' 960 = 960 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=960 ++ echo -e '\033[32mLatest checkpoint iteration: 960\033[0m\n' +Latest checkpoint iteration: 960 + ++ I=960 ++ [[ 960 =~ ^[0-9]+$ ]] +++ printf %07d 960 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960/iter_0000960 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960/iter_0000960/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960/iter_960-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_960-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_960-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_960-hf true false bf16 /tmp/mcore2hf_daemon_960/mcore2hf-config.jmcWtl 960 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_960-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 33383 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_daemon_960/mcore2hf-config.jmcWtl --hf-dir /tmp/mcore2hf_daemon_960/mcore2hf-config.jmcWtl --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 524288 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_960-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 960 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1014 02:20:55.732000 752973 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1014 02:20:55.732000 752973 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 960 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_daemon_960/mcore2hf-config.jmcWtl + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_960-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_daemon_960/mcore2hf-config.jmcWtl + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.132 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_jmcWtl/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_jmcWtl/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_jmcWtl.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 at iteration 960 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/load_view960 [ t 1/1, p 1/1 ] at iteration 960 +[convert] synchronizer created in 377.87s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=-0.00049591064453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10302734375 mean=0.00168609619140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0791015625 mean=-0.0001163482666015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1064453125 mean=-4.601478576660156e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0849609375 mean=-0.00021457672119140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08056640625 mean=0.000827789306640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08837890625 mean=-0.00022029876708984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07861328125 mean=0.001434326171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0966796875 mean=-0.0002155303955078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07470703125 mean=-0.000782012939453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.091796875 mean=-7.534027099609375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06298828125 mean=-0.00021266937255859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09375 mean=-7.62939453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=0.0001735687255859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08837890625 mean=4.315376281738281e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.056884765625 mean=-0.0004520416259765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08447265625 mean=6.008148193359375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05517578125 mean=-0.0006561279296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08251953125 mean=2.6941299438476562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06640625 mean=-0.0004596710205078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00019931793212890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05517578125 mean=-0.00069427490234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=0.000110626220703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=-0.0010528564453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00015354156494140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047119140625 mean=-0.000965118408203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.00022602081298828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0498046875 mean=-0.000637054443359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.00029754638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04833984375 mean=-0.000530242919921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.000316619873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047119140625 mean=-0.00142669677734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.000270843505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0478515625 mean=-0.0020599365234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.00055694580078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0498046875 mean=-0.0016632080078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.0003509521484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.051025390625 mean=-0.0013275146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.0003719329833984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=-0.002197265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0556640625 mean=0.0002956390380859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044189453125 mean=-0.0030975341796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.0004367828369140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037841796875 mean=-0.002288818359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.0004253387451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03564453125 mean=-0.0025177001953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.000377655029296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0341796875 mean=-0.00164031982421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.0003070831298828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03173828125 mean=-0.0015411376953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.00041961669921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03076171875 mean=-0.00180816650390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047607421875 mean=0.0003814697265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02734375 mean=-0.00106048583984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.00052642822265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0244140625 mean=-0.000782012939453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.00081634521484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0257568359375 mean=-0.000946044921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.000530242919921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0267333984375 mean=-0.000530242919921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046142578125 mean=0.00012874603271484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025634765625 mean=-0.0004673004150390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=8.392333984375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0279541015625 mean=-6.866455078125e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.000152587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027099609375 mean=-7.915496826171875e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=4.7206878662109375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032470703125 mean=0.0002918243408203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.059326171875 mean=4.3392181396484375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=0.00098419189453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05908203125 mean=0.00010013580322265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=0.00131988525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0576171875 mean=0.00010776519775390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037353515625 mean=0.0014801025390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05615234375 mean=6.341934204101562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037109375 mean=0.001190185546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.0001049041748046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02587890625 mean=0.0003604888916015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=9.34600830078125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0245361328125 mean=0.00023174285888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0439453125 mean=0.0003223419189453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021240234375 mean=-0.00054931640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=0.00017261505126953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0203857421875 mean=-0.000171661376953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=0.0004253387451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02197265625 mean=-0.0003643035888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=9.012222290039062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.000804901123046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.00029754638671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021240234375 mean=-0.000637054443359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.00017833709716796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.0010223388671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.033935546875 mean=-9.059906005859375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0198974609375 mean=-0.000431060791015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.033935546875 mean=-0.00013256072998046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021728515625 mean=-0.0010528564453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=-0.000125885009765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0240478515625 mean=-0.000720977783203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03955078125 mean=1.6450881958007812e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0242919921875 mean=-0.00107574462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.040283203125 mean=0.00018978118896484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031005859375 mean=-0.00135040283203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037353515625 mean=8.869171142578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.053466796875 mean=-0.0024261474609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=9.012222290039062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041259765625 mean=-0.00194549560546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.043212890625 mean=-0.00012683868408203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044189453125 mean=0.0026397705078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.94s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_960-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 77.81s +Conversion finished in 458.0295913219452 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_960-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work/iter_960-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_daemon_960/mcore2hf-config.jmcWtl ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/hf_export_daemon_stage16a.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/hf_export_daemon_stage16a.sh new file mode 100644 index 0000000000000000000000000000000000000000..749696841631e17c036b2dfb8680d0b576421314 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/hf_export_logs/hf_export_daemon_stage16a.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# HF auto-export daemon for stage16a (512K, CP32) — for later ckpt merging. +# Every EXPORT_INTERVAL iterations of the training ckpt dir, produce a model-only +# HF export and upload it to S3: hardlink-preserve the ckpt (SSD, instant), +# convert on CPU, validate, rclone to s3://yulan/$S3_PREFIX/iter_XXXXXXX-hf/, +# verify byte-identical, then delete local copies. Also exports the FINAL +# iteration when training exits. +# Zero interaction with the training process (read-only on ckpts, CPU-only conversion). +set -uo pipefail + +S16A=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a +CKPT_DIR=$S16A/checkpoints +WORK=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage16a_daemon_work +S3_PREFIX=f3-plain-shortconv-adamh-stage16a-cp32-512k-hf +DONE_DIR=$WORK/done +EXPORT_INTERVAL=80 +MIN_SSD_FREE_BYTES=350000000000 # 350G: stop exporting below this (transient work dir) +FINAL_ITER=2384 +CONVERTER_DIR=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor + +mkdir -p "$WORK" "$DONE_DIR" +log() { echo "[$(date '+%F %T')] $*"; } + +ckpt_complete() { + local d=$1 + [ -d "$d" ] || return 1 + [ -f "$d/.metadata" ] || return 1 + [ "$(ls "$d" | wc -l)" -ge 131 ] || return 1 + ! ls "$d" | grep -qiE 'temp|incomplete' +} + +ssd_free_bytes() { df -B1 /mnt/ssd | tail -1 | awk '{print $4}'; } + +s3_bytes() { + source /root/debug_2440/rclone_env.sh + rclone size "s3:yulan/$S3_PREFIX/iter_$1-hf/" 2>/dev/null | grep -oE '[0-9]+ Byte' | grep -oE '^[0-9]+' || echo 0 +} + +export_one() { + local iter=$1 + local tag=$(printf "%07d" "$iter") + local s3_dir="s3:yulan/$S3_PREFIX/iter_${tag}-hf" + [ -e "$DONE_DIR/${tag}.done" ] && return 0 + + ckpt_complete "$CKPT_DIR/iter_$tag" || { log "iter $iter: ckpt incomplete, skip"; return 1; } + local free=$(ssd_free_bytes) + [ "$free" -lt "$MIN_SSD_FREE_BYTES" ] && { log "iter $iter: SSD free ${free}B < guard, STOP exporting"; return 2; } + + log "iter $iter: preserving ckpt (hardlinks)..." + rm -rf "$WORK/iter_${tag}_preserve" "$WORK/load_view${iter}" + cp -al "$CKPT_DIR/iter_$tag" "$WORK/iter_${tag}_preserve" || { log "iter $iter: preserve FAILED"; return 1; } + mkdir -p "$WORK/load_view${iter}" + ln -sfn "$WORK/iter_${tag}_preserve" "$WORK/load_view${iter}/iter_$tag" + echo "$iter" > "$WORK/load_view${iter}/latest_checkpointed_iteration.txt" + + log "iter $iter: converting to HF (CPU)..." + cd "$CONVERTER_DIR" + env USE_CUDA=false \ + PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud \ + AUTO_REPAIR_GDN_CHECKPOINT=0 \ + WORKSPACE_TMP_DIR=/tmp/mcore2hf_daemon_$iter \ + HF_SAVE_ROOT=$WORK \ + EXTRA_ARGS="--gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo" \ + MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml \ + bash scripts/mcore_gdn_moe/launch-mcore2hf.sh "$WORK/load_view${iter}" --iter "$iter" \ + > "$WORK/convert_iter${iter}.log" 2>&1 + local rc=$? + + if [ $rc -ne 0 ] || ! grep -q "Conversion completed" "$WORK/convert_iter${iter}.log"; then + log "iter $iter: CONVERSION FAILED (rc=$rc), keeping preserve for debug" + return 1 + fi + # validate output + local hfdir="$WORK/iter_${iter}-hf" + local nfiles=$(find "$hfdir" -type f ! -name hf2mcore.log | wc -l) + local nmap=$(python3 -c "import json;print(len(json.load(open('$hfdir/model.safetensors.index.json'))['weight_map']))" 2>/dev/null || echo 0) + if [ "$nfiles" != "11" ] || [ "$nmap" != "32226" ]; then + log "iter $iter: VALIDATION FAILED (files=$nfiles weight_map=$nmap), keeping preserve" + return 1 + fi + # upload to S3 and verify byte-identical before any cleanup + log "iter $iter: uploading to $s3_dir ..." + source /root/debug_2440/rclone_env.sh + rclone copy "$hfdir/" "$s3_dir/" --exclude hf2mcore.log --transfers 4 --checkers 4 \ + --s3-chunk-size 128M --s3-upload-concurrency 8 >> "$WORK/convert_iter${iter}.log" 2>&1 + if [ $? -ne 0 ]; then + log "iter $iter: S3 UPLOAD FAILED, keeping local copy at $hfdir" + return 1 + fi + local local_bytes=$(find "$hfdir" -type f ! -name hf2mcore.log -printf "%s\n" | awk '{s+=$1} END{print s}') + local remote_bytes=$(s3_bytes "$tag") + if [ "$local_bytes" != "$remote_bytes" ]; then + log "iter $iter: S3 VERIFY MISMATCH local=$local_bytes remote=$remote_bytes, keeping local copy" + return 1 + fi + rm -rf "$hfdir" "$WORK/iter_${tag}_preserve" "$WORK/load_view${iter}" /tmp/mcore2hf_daemon_$iter + touch "$DONE_DIR/${tag}.done" + log "iter $iter: exported -> $s3_dir (verified $remote_bytes bytes), local cleaned" + return 0 +} + +log "daemon start: ckpt=$CKPT_DIR interval=$EXPORT_INTERVAL s3=s3://yulan/$S3_PREFIX" +while true; do + if [ -f "$CKPT_DIR/latest_checkpointed_iteration.txt" ]; then + iter=$(cat "$CKPT_DIR/latest_checkpointed_iteration.txt" 2>/dev/null || echo 0) + iter=${iter//[^0-9]/} + [ -z "$iter" ] && iter=0 + target=$(( iter / EXPORT_INTERVAL * EXPORT_INTERVAL )) + if [ "$target" -ge "$EXPORT_INTERVAL" ]; then + export_one "$target" + fi + # training finished: export final iteration once, then exit + if ! pgrep -f 'pretrain_gp[t].py' > /dev/null; then + log "no training process detected; latest iter=$iter" + export_one "$iter" && log "final iter $iter exported; daemon exit" && exit 0 + sleep 300 # final export failed; retry later + fi + fi + sleep 60 +done diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node0.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node0.sh new file mode 100644 index 0000000000000000000000000000000000000000..4e2687071c1e96666fc43d2d59cd4a51b8d2ae14 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node0.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Stage16a (two-phase plan, run A): 40B tokens normal attention on part1_40B split. +# 512K seq length, loads stage15 final weights (--continual-pretrain resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 32 docs x 524288 = 16,777,216 tokens/step; 76288 samples = 2384 iters. +# part1 = 39.9998B tokens (every source cut at the same doc ratio, see split_20261010/split_report.json). +# master-port 35873. TP1 x PP1 x CP16 x EP8, cp-comm-type p2p (TE has no thd support for a2a+p2p; p2p ring attention has no head-divisibility constraint; 2xCP zigzag chunking convention unchanged). Patches: packing collator + arguments + gdn_pre_gdr whitelists allow CP16. EP8 keeps stage15 optimizer ckpt compatible. CE chunks 32. +# After completion, stage16b resumes from this run's final ckpt with sparse attention. +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 0 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node1.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node1.sh new file mode 100644 index 0000000000000000000000000000000000000000..f938c4e9260b435e1d10496cf1b3865175ba344b --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node1.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (--continual-pretrain resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 1 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node2.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node2.sh new file mode 100644 index 0000000000000000000000000000000000000000..6f90964fe4b83819a81388146905afc439e9cd77 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node2.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (--continual-pretrain resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 2 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node3.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node3.sh new file mode 100644 index 0000000000000000000000000000000000000000..c014577a5967d035604aab7316d544a165f12b99 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node3.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (--continual-pretrain resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 3 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node4.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node4.sh new file mode 100644 index 0000000000000000000000000000000000000000..2cf13a8bcf04931c3978d5dad4fc4568a833f777 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node4.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (--continual-pretrain resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 4 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node5.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node5.sh new file mode 100644 index 0000000000000000000000000000000000000000..44c17e45ba2a7f10fbca0f52988945601b590b1c --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node5.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (--continual-pretrain resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 5 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node6.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node6.sh new file mode 100644 index 0000000000000000000000000000000000000000..5cf16379f6258c8a59102b12679a6eee00cc9100 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node6.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (--continual-pretrain resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 6 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node7.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node7.sh new file mode 100644 index 0000000000000000000000000000000000000000..f51ad6f1b096a92ad01562defc75fc64af49dcff --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/launch_node7.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (--continual-pretrain resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 7 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node0.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node0.log new file mode 100644 index 0000000000000000000000000000000000000000..6d80f285ac7aeaf56400b32b3ba2b125a6b8460f --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node0.log @@ -0,0 +1,4664 @@ +W1011 02:40:48.439000 894734 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:48.439000 894734 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:48.591000 894736 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:48.591000 894736 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:48.653000 894733 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:48.653000 894733 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:48.790000 894731 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:48.790000 894731 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +using world size: 64, data-parallel size: 4, context-parallel size: 16, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:HuggingFaceTokenizer +Number of virtual stages per pipeline stage: None +[gdn-fused-split-forward] optimizer overlap enabled: param-gather=ddp-extra-param-bridge grad-reduce=standard-autograd-hooks +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/training/utils.py:548: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn(message) +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... soft + activation_func_clamp_value ..................... 7.0 + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ False + adam_eps ........................................ 1e-15 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... True + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... True + auto_generate_cu_seqlens ........................ True + barrier_with_L1_time ............................ False + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ True + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 16 + continual_pretrain .............................. True + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... True + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. False + cross_entropy_fusion_impl ....................... linear_triton + cross_entropy_loss_fusion ....................... True + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 4 + data_path ....................................... ['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. cyclic + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. 40000000 + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... False + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 120 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... obfd + document_packing_dp_size ........................ None + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 524288 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 99999999 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... True + exit_signal_handler ............................. True + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... True + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ 30.0 + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... True + gdn_gated_norm_backend .......................... linghe + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 32 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... True + hybrid_attention_ratio .......................... 0.0625 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.5 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 64 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. True + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 32 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... global-token + load ............................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + load_complemental_dataset ....................... ['loss_mask'] + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... ['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] + log_interval .................................... 1 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ 10 + log_moe_activations ............................. True + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... ['attention::linear_qkv', 'attention::o_proj', 'mlp::linear_fc1', 'mlp::linear_fc2', 'input_layernorm', 'pre_mlp_layernorm', 'embedding', 'lm_head', 'gated_delta::in_proj', 'gated_delta::in_proj_bias', 'gated_delta::conv1d', 'gated_delta::conv1d_bias', 'gated_delta::dt_bias', 'gated_delta::A_log', 'gated_delta::out_norm', 'gated_delta::out_norm_bias', 'gated_delta::out_proj', 'gated_delta::out_proj_bias'] + log_params_norm ................................. True + log_per_layer_grad_rms .......................... True + log_per_module_grad_rms ......................... True + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_update_rms .................................. True + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0005 + lr_decay_iters .................................. None + lr_decay_samples ................................ 76288 + lr_decay_style .................................. constant + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 0.0005 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.05 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. True + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... True + moe_router_gemm_backend ......................... te + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. global_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... True + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... gated_delta_net + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. 0 + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ True + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 4 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... True + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... False + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. 8 + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... full + recompute_method ................................ block + recompute_modules ............................... None + recompute_num_layers ............................ 56 + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... True + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit + sample_rate ..................................... 1.0 + save ............................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + save_after_load ................................. False + save_interval ................................... 40 + save_retain_interval ............................ 1000000000 + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 524288 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... 999,1,0 + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... 0 + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... all + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... None + train_samples ................................... 76288 + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. True + use_fused_weighted_squared_relu ................. False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. stage15_115B_8node_gdnlralpha_off_20261003a + wandb_project ................................... qwen2.5_3b_mom_cpt + wandb_save_dir .................................. /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 64 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 8 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank0]:[W1011 02:41:49.696643684 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank0]:[W1011 02:41:49.696659063 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 1 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank1]:[W1011 02:41:49.696784910 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank1]:[W1011 02:41:49.696799462 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 6 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank6]:[W1011 02:41:49.697479744 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank6]:[W1011 02:41:49.697529087 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 5 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank5]:[W1011 02:41:49.697712952 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank5]:[W1011 02:41:49.697725195 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 4 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 2 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 7 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank4]:[W1011 02:41:49.697849764 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank4]:[W1011 02:41:49.697862261 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank2]:[W1011 02:41:49.697889598 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank2]:[W1011 02:41:49.697901689 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 3 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank7]:[W1011 02:41:49.698020491 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank7]:[W1011 02:41:49.698039973 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank3]:[W1011 02:41:49.698056527 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank3]:[W1011 02:41:49.698067189 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank1]:[W1011 02:41:52.806637277 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank7]:[W1011 02:41:52.812992494 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank2]:[W1011 02:41:52.820240851 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank3]:[W1011 02:41:52.820884678 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank4]:[W1011 02:41:52.823012422 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank6]:[W1011 02:41:52.823326035 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank5]:[W1011 02:41:52.823932174 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.131 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank0]:[W1011 02:41:52.946744963 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +NCCL version 2.27.3+cuda12.9 +[rank6]:[W1011 02:41:52.590701651 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank7]:[W1011 02:41:52.590709157 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank0]:[W1011 02:41:52.590744002 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank3]:[W1011 02:41:52.590778472 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank6]:[W1011 02:41:52.590793418 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank7]:[W1011 02:41:52.590829945 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank4]:[W1011 02:41:52.590823457 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank0]:[W1011 02:41:52.590876696 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank3]:[W1011 02:41:52.590903463 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank1]:[W1011 02:41:52.590924985 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank4]:[W1011 02:41:52.590958776 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank5]:[W1011 02:41:52.590963946 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank1]:[W1011 02:41:52.590997684 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank5]:[W1011 02:41:52.591030720 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank2]:[W1011 02:41:52.591084652 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank2]:[W1011 02:41:52.591125935 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +>>> done with compiling and loading fused kernels. Compilation time: 1.671 seconds +time to initialize megatron (seconds): 380.764 +[after megatron is initialized] datetime: 2026-10-11 02:47:10 +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5,569,877,120 + > number of trainable parameters on (tensor, pipeline) model parallel rank (0, 0): 5,569,877,120 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, defer_grad_checks=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_megatron_fsdp=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache=False, nccl_ub=False, fsdp_double_buffer=False, outer_dp_sharding_strategy='no_shard', disable_symmetric_registration=False, delay_wgrad_compute=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 28 +Params for bucket 1 (202899456 elements, 202899456 padded size): + module.output_layer.weight torch.Size([99072, 2048]) +Params for bucket 2 (40222848 elements, 40222848 padded size): + module.decoder.final_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.final_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.55.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.54.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.54.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.54.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.54.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.55.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.55.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.55.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.54.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.54.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.final_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.55.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.55.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.55.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.54.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.55.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.54.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.54.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.54.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.55.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.55.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.54.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.55.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.54.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.54.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.55.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.55.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.54.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.54.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.final_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.55.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.55.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.55.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.54.self_attention.b_proj.weight torch.Size([32, 2048]) +Params for bucket 3 (41666688 elements, 41666688 padded size): + module.decoder.layers.53.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.53.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.53.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.52.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.52.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.52.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.53.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.52.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.51.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.51.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.53.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.53.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.53.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.52.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.52.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.51.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.51.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.53.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.53.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.52.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.52.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.52.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.53.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.52.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.52.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.52.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.52.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.53.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.53.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.53.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.53.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.52.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.52.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.53.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.53.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.52.pre_mlp_layernorm.norm.weight torch.Size([2048]) +Params for bucket 4 (43304064 elements, 43304064 padded size): + module.decoder.layers.51.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.51.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.50.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.50.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.49.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.51.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.50.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.50.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.51.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.50.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.49.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.51.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.50.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.50.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.50.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.49.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.51.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.50.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.50.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.50.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.49.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.51.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.51.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.51.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.51.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.50.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.51.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.50.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.50.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.50.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.49.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.51.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.50.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) +Params for bucket 5 (40014912 elements, 40014976 padded size): + module.decoder.layers.48.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.47.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.47.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.49.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.49.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.48.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.48.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.47.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.49.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.48.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.47.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.47.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.47.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.48.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.48.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.48.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.47.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.48.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.47.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.47.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.49.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.48.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.48.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.49.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.47.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.47.self_attention.b_proj.weight torch.Size([32, 2048]) +Params for bucket 6 (41133120 elements, 41133184 padded size): + module.decoder.layers.47.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.46.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.46.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.45.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.45.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.45.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.46.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.45.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.45.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.45.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.45.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.47.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.46.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.46.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.46.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.46.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.46.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.45.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.45.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.45.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.47.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.46.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.45.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.47.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.45.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.45.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.45.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.45.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.45.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.47.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.46.input_layernorm.gate_up.bias torch.Size([2048]) +Params for bucket 7 (41666688 elements, 41666688 padded size): + module.decoder.layers.44.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.43.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.43.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.43.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.44.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.44.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.44.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.44.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.43.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.43.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.43.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.44.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.43.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.43.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.44.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.43.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.44.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.44.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.44.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.43.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.43.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.43.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.44.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.43.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.44.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.44.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.43.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.42.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.42.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.44.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.44.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.44.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.43.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.43.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.42.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.42.pre_mlp_layernorm.norm.weight torch.Size([2048]) +Params for bucket 8 (41345216 elements, 41345280 padded size): + module.decoder.layers.42.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.41.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.41.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.41.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.42.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.42.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.42.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.42.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.41.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.41.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.40.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.42.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.41.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.41.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.41.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.40.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.42.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.41.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.40.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.40.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.42.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.42.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.41.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.41.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.41.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.41.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.40.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.40.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.40.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.42.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.42.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.41.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.40.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.42.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.41.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.41.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.40.mlp.router.weight torch.Size([192, 2048]) +Params for bucket 9 (41201792 elements, 41201792 padded size): + module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.39.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.38.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.38.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.38.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.39.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.39.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.39.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.39.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.40.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.39.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.38.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.40.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.39.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.40.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.39.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.39.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.38.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.40.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.40.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.39.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.39.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.38.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.40.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.40.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.39.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.39.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.39.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.38.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.39.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.39.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.38.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.38.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.38.self_attention.z_proj.weight torch.Size([2048, 2048]) +Params for bucket 10 (40222848 elements, 40222848 padded size): + module.decoder.layers.37.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.36.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.36.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.38.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.37.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.37.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.36.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.38.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.38.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.38.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.37.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.37.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.36.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.36.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.36.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.38.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.37.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.36.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.36.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.36.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.37.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.37.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.37.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.37.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.36.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.38.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.37.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.37.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.36.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.36.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.36.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.36.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.37.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.37.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.36.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.37.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.37.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.36.self_attention.out_norm.weight torch.Size([64]) +Params for bucket 11 (43763840 elements, 43763840 padded size): + module.decoder.layers.35.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.35.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.35.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.35.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.34.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.34.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.35.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.34.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.35.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.34.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.34.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.34.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.35.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.35.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.34.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.34.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.33.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.33.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.35.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.35.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.34.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.34.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.33.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.33.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.35.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.35.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.35.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.34.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.34.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.34.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.34.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.35.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.35.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.34.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.35.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.34.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) +Params for bucket 12 (41345216 elements, 41345280 padded size): + module.decoder.layers.33.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.32.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.32.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.31.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.33.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.33.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.32.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.32.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.32.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.31.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.31.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.33.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.32.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.32.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.31.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.33.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.32.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.33.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.32.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.32.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.32.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.31.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.33.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.32.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.32.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.33.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.33.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.33.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.33.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.32.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.31.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.31.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.33.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.32.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.32.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.31.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.31.self_attention.a_proj.weight torch.Size([32, 2048]) +Params for bucket 13 (41201792 elements, 41201792 padded size): + module.decoder.layers.30.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.30.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.30.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.29.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.29.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.29.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.31.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.30.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.29.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.29.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.29.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.31.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.30.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.30.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.30.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.30.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.29.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.29.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.31.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.30.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.31.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.31.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.30.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.31.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.30.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.30.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.29.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.30.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.31.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.30.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.30.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.30.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.29.self_attention.a_proj.weight torch.Size([32, 2048]) +Params for bucket 14 (40222848 elements, 40222848 padded size): + module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.28.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.28.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.27.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.27.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.28.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.27.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.27.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.27.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.28.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.28.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.28.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.27.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.29.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.29.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.28.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.28.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.28.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.27.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.27.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.27.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.27.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.29.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.29.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.29.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.28.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.27.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.28.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.28.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.28.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.28.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.29.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.28.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.27.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.27.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.28.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.27.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.27.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.27.input_layernorm.gate_down.weight torch.Size([16, 2048]) +Params for bucket 15 (43763840 elements, 43763840 padded size): + module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.26.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.26.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.26.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.26.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.25.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.26.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.26.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.25.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.25.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.26.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.25.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.26.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.26.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.25.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.25.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.25.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.26.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.25.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.25.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.24.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.24.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.26.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.26.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.25.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.24.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.24.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.26.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.26.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.25.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.25.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.25.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.25.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.26.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.26.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.25.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.25.input_layernorm.gate_down.weight torch.Size([16, 2048]) +Params for bucket 16 (41206912 elements, 41206912 padded size): + module.decoder.layers.24.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.23.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.24.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.23.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.22.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.24.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.24.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.23.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.22.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.24.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.23.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.23.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.22.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.24.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.23.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.23.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.22.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.22.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.22.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.24.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.23.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.22.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.22.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.24.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.23.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.23.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.24.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.24.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.24.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.24.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.22.self_attention.conv1d.weight torch.Size([3072, 1, 4]) +Params for bucket 17 (43022336 elements, 43022336 padded size): + module.decoder.layers.22.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.21.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.20.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.20.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.22.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.21.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.20.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.22.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.21.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.21.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.20.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.21.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.21.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.21.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.22.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.21.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.22.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.21.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.20.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.22.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.22.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.21.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.20.self_attention.linear_qkv.bias torch.Size([5120]) +Params for bucket 18 (40222848 elements, 40222848 padded size): + module.decoder.layers.19.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.19.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.19.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.18.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.18.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.18.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.18.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.20.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.20.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.19.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.18.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.18.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.20.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.19.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.19.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.19.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.19.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.18.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.19.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.18.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.18.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.20.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.19.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.19.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.18.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.19.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.19.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.18.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.18.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.18.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.19.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.18.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.18.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.19.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.19.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.18.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) +Params for bucket 19 (41666688 elements, 41666688 padded size): + module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.17.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.16.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.16.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.17.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.16.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.16.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.17.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.17.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.17.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.16.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.16.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.17.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.16.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.16.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.16.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.17.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.17.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.16.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.15.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.17.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.17.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.16.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.16.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.16.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.16.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.15.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.15.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.17.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.16.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.15.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.17.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.17.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.17.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.17.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.17.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.16.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) +Params for bucket 20 (41345216 elements, 41345280 padded size): + module.decoder.layers.15.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.15.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.15.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.15.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.14.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.13.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.15.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.14.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.15.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.14.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.14.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.13.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.15.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.15.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.14.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.14.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.13.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.13.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.15.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.14.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.14.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.14.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.13.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.15.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.14.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.14.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.13.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.13.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.13.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.15.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.14.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.13.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.15.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.14.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.14.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.14.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.14.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) +Params for bucket 21 (41063488 elements, 41063552 padded size): + module.decoder.layers.13.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.13.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.12.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.12.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.11.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.11.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.11.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.13.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.12.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.13.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.12.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.11.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.13.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.12.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.12.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.11.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.12.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.11.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.13.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.12.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.12.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.11.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.13.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.12.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.11.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.11.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.11.self_attention.z_proj.weight torch.Size([2048, 2048]) +Params for bucket 22 (40222848 elements, 40222848 padded size): + module.decoder.layers.10.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.10.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.10.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.10.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.9.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.11.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.10.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.10.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.9.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.9.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.9.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.11.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.11.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.11.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.10.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.9.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.11.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.10.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.10.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.9.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.9.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.9.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.9.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.10.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.9.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.9.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.11.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.10.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.10.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.10.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.9.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.9.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.10.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.10.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.9.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.10.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.9.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.9.input_layernorm.gate_down.weight torch.Size([16, 2048]) +Params for bucket 23 (43763840 elements, 43763840 padded size): + module.decoder.layers.8.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.7.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.8.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.8.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.7.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.6.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.8.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.7.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.7.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.6.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.8.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.8.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.8.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.7.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.6.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.8.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.8.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.8.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.7.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.7.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.7.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.8.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.7.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.7.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.7.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.6.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.8.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.8.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.8.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.8.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.7.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.8.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.7.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.7.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.7.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.7.input_layernorm.norm.weight torch.Size([2048]) +Params for bucket 24 (41345216 elements, 41345280 padded size): + module.decoder.layers.6.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.5.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.5.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.4.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.6.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.5.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.4.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.6.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.5.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.5.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.5.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.4.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.6.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.5.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.5.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.5.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.4.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.4.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.4.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.6.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.6.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.6.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.6.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.5.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.4.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.4.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.6.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.5.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.5.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.5.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.5.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.6.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.5.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.4.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.6.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.6.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.5.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) +Params for bucket 25 (41201792 elements, 41201792 padded size): + module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.2.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.4.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.3.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.2.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.2.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.3.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.3.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.3.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.2.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.3.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.3.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.2.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.2.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.4.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.3.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.3.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.2.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.2.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.4.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.3.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.3.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.3.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.4.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.4.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.4.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.3.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.2.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.2.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.4.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.3.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.3.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.3.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.3.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) +Params for bucket 26 (40259648 elements, 40259712 padded size): + module.decoder.layers.2.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.2.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.1.mlp.linear_fc2.weight torch.Size([2048, 4800]) + module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.1.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.1.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.1.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.2.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.2.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.2.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.2.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.1.mlp.linear_fc1.weight torch.Size([9600, 2048]) + module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.1.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.1.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.1.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.1.self_attention.b_proj.weight torch.Size([32, 2048]) +Params for bucket 27 (40116224 elements, 40116224 padded size): + module.decoder.layers.1.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.1.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.1.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.1.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.0.mlp.linear_fc2.weight torch.Size([2048, 4800]) + module.decoder.layers.0.mlp.linear_fc1.weight torch.Size([9600, 2048]) + module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.0.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.1.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.1.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.1.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.0.self_attention.out_proj.weight torch.Size([2048, 2048]) +Params for bucket 28 (213598272 elements, 213598336 padded size): + module.decoder.layers.0.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.0.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.0.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.0.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.0.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.0.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.embedding.word_embeddings.weight torch.Size([99072, 2048]) + module.decoder.layers.0.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.0.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.0.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.0.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.0.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.0.input_layernorm.norm.weight torch.Size([2048]) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (3136 elements, 6272 padded size): + module.decoder.layers.17.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.51.self_attention.A_log torch.Size([32]) + module.decoder.layers.36.self_attention.A_log torch.Size([32]) + module.decoder.layers.15.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.9.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.5.self_attention.A_log torch.Size([32]) + module.decoder.layers.52.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.33.self_attention.A_log torch.Size([32]) + module.decoder.layers.1.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.0.self_attention.A_log torch.Size([32]) + module.decoder.layers.0.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.45.self_attention.A_log torch.Size([32]) + module.decoder.layers.38.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.22.self_attention.A_log torch.Size([32]) + module.decoder.layers.19.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.11.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.3.self_attention.A_log torch.Size([32]) + module.decoder.layers.26.self_attention.A_log torch.Size([32]) + module.decoder.layers.13.self_attention.A_log torch.Size([32]) + module.decoder.layers.43.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.39.self_attention.A_log torch.Size([32]) + module.decoder.layers.32.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.31.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.28.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.54.self_attention.A_log torch.Size([32]) + module.decoder.layers.47.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.40.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.25.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.41.self_attention.A_log torch.Size([32]) + module.decoder.layers.7.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.35.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.50.self_attention.A_log torch.Size([32]) + module.decoder.layers.42.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.8.self_attention.A_log torch.Size([32]) + module.decoder.layers.4.self_attention.A_log torch.Size([32]) + module.decoder.layers.2.self_attention.A_log torch.Size([32]) + module.decoder.layers.17.self_attention.A_log torch.Size([32]) + module.decoder.layers.15.self_attention.A_log torch.Size([32]) + module.decoder.layers.55.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.37.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.6.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.1.self_attention.A_log torch.Size([32]) + module.decoder.layers.38.self_attention.A_log torch.Size([32]) + module.decoder.layers.34.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.19.self_attention.A_log torch.Size([32]) + module.decoder.layers.16.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.11.self_attention.A_log torch.Size([32]) + module.decoder.layers.9.self_attention.A_log torch.Size([32]) + module.decoder.layers.53.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.31.self_attention.A_log torch.Size([32]) + module.decoder.layers.24.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.14.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.28.self_attention.A_log torch.Size([32]) + module.decoder.layers.18.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.10.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.44.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.40.self_attention.A_log torch.Size([32]) + module.decoder.layers.25.self_attention.A_log torch.Size([32]) + module.decoder.layers.43.self_attention.A_log torch.Size([32]) + module.decoder.layers.32.self_attention.A_log torch.Size([32]) + module.decoder.layers.30.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.7.self_attention.A_log torch.Size([32]) + module.decoder.layers.45.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.35.self_attention.A_log torch.Size([32]) + module.decoder.layers.27.self_attention.A_log torch.Size([32]) + module.decoder.layers.42.self_attention.A_log torch.Size([32]) + module.decoder.layers.27.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.51.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.36.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.5.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.33.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.55.self_attention.A_log torch.Size([32]) + module.decoder.layers.52.self_attention.A_log torch.Size([32]) + module.decoder.layers.37.self_attention.A_log torch.Size([32]) + module.decoder.layers.22.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.6.self_attention.A_log torch.Size([32]) + module.decoder.layers.3.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.34.self_attention.A_log torch.Size([32]) + module.decoder.layers.26.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.16.self_attention.A_log torch.Size([32]) + module.decoder.layers.13.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.53.self_attention.A_log torch.Size([32]) + module.decoder.layers.39.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.24.self_attention.A_log torch.Size([32]) + module.decoder.layers.14.self_attention.A_log torch.Size([32]) + module.decoder.layers.54.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.18.self_attention.A_log torch.Size([32]) + module.decoder.layers.10.self_attention.A_log torch.Size([32]) + module.decoder.layers.44.self_attention.A_log torch.Size([32]) + module.decoder.layers.41.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.29.self_attention.A_log torch.Size([32]) + module.decoder.layers.30.self_attention.A_log torch.Size([32]) + module.decoder.layers.29.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.50.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.47.self_attention.A_log torch.Size([32]) + module.decoder.layers.8.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.4.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.2.self_attention.dt_bias torch.Size([32]) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 54 +Params for bucket 1 (75497472 elements, 75497472 padded size): + module.decoder.layers.55.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.55.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 2 (75497472 elements, 75497472 padded size): + module.decoder.layers.54.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.54.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 3 (75497472 elements, 75497472 padded size): + module.decoder.layers.53.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.53.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 4 (75497472 elements, 75497472 padded size): + module.decoder.layers.52.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.52.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 5 (75497472 elements, 75497472 padded size): + module.decoder.layers.51.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.51.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 6 (75497472 elements, 75497472 padded size): + module.decoder.layers.50.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.50.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 7 (75497472 elements, 75497472 padded size): + module.decoder.layers.49.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.49.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 8 (75497472 elements, 75497472 padded size): + module.decoder.layers.48.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.48.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 9 (75497472 elements, 75497472 padded size): + module.decoder.layers.47.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.47.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 10 (75497472 elements, 75497472 padded size): + module.decoder.layers.46.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.46.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 11 (75497472 elements, 75497472 padded size): + module.decoder.layers.45.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.45.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 12 (75497472 elements, 75497472 padded size): + module.decoder.layers.44.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.44.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 13 (75497472 elements, 75497472 padded size): + module.decoder.layers.43.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.43.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 14 (75497472 elements, 75497472 padded size): + module.decoder.layers.42.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.42.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 15 (75497472 elements, 75497472 padded size): + module.decoder.layers.41.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.41.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 16 (75497472 elements, 75497472 padded size): + module.decoder.layers.40.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.40.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 17 (75497472 elements, 75497472 padded size): + module.decoder.layers.39.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.39.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 18 (75497472 elements, 75497472 padded size): + module.decoder.layers.38.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.38.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 19 (75497472 elements, 75497472 padded size): + module.decoder.layers.37.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.37.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 20 (75497472 elements, 75497472 padded size): + module.decoder.layers.36.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.36.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 21 (75497472 elements, 75497472 padded size): + module.decoder.layers.35.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.35.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 22 (75497472 elements, 75497472 padded size): + module.decoder.layers.34.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.34.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 23 (75497472 elements, 75497472 padded size): + module.decoder.layers.33.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.33.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 24 (75497472 elements, 75497472 padded size): + module.decoder.layers.32.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.32.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 25 (75497472 elements, 75497472 padded size): + module.decoder.layers.31.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.31.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 26 (75497472 elements, 75497472 padded size): + module.decoder.layers.30.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.30.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 27 (75497472 elements, 75497472 padded size): + module.decoder.layers.29.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.29.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 28 (75497472 elements, 75497472 padded size): + module.decoder.layers.28.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.28.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 29 (75497472 elements, 75497472 padded size): + module.decoder.layers.27.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.27.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 30 (75497472 elements, 75497472 padded size): + module.decoder.layers.26.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.26.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 31 (75497472 elements, 75497472 padded size): + module.decoder.layers.25.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.25.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 32 (75497472 elements, 75497472 padded size): + module.decoder.layers.24.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.24.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 33 (75497472 elements, 75497472 padded size): + module.decoder.layers.23.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.23.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 34 (75497472 elements, 75497472 padded size): + module.decoder.layers.22.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.22.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 35 (75497472 elements, 75497472 padded size): + module.decoder.layers.21.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.21.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 36 (75497472 elements, 75497472 padded size): + module.decoder.layers.20.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.20.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 37 (75497472 elements, 75497472 padded size): + module.decoder.layers.19.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.19.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 38 (75497472 elements, 75497472 padded size): + module.decoder.layers.18.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.18.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 39 (75497472 elements, 75497472 padded size): + module.decoder.layers.17.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.17.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 40 (75497472 elements, 75497472 padded size): + module.decoder.layers.16.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.16.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 41 (75497472 elements, 75497472 padded size): + module.decoder.layers.15.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.15.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 42 (75497472 elements, 75497472 padded size): + module.decoder.layers.14.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.14.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 43 (75497472 elements, 75497472 padded size): + module.decoder.layers.13.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.13.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 44 (75497472 elements, 75497472 padded size): + module.decoder.layers.12.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.12.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 45 (75497472 elements, 75497472 padded size): + module.decoder.layers.11.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.11.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 46 (75497472 elements, 75497472 padded size): + module.decoder.layers.10.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.10.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 47 (75497472 elements, 75497472 padded size): + module.decoder.layers.9.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.9.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 48 (75497472 elements, 75497472 padded size): + module.decoder.layers.8.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.8.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 49 (75497472 elements, 75497472 padded size): + module.decoder.layers.7.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.7.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 50 (75497472 elements, 75497472 padded size): + module.decoder.layers.6.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.6.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 51 (75497472 elements, 75497472 padded size): + module.decoder.layers.5.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.5.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 52 (75497472 elements, 75497472 padded size): + module.decoder.layers.4.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.4.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 53 (75497472 elements, 75497472 padded size): + module.decoder.layers.3.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.3.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 54 (75497472 elements, 75497472 padded size): + module.decoder.layers.2.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.2.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +[GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.0, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.0, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.0, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.0, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.0, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.0, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): FusedLinearCrossEntropyColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +)] +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0005, min_lr=0.0005, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, corrected_weight_decay=True, fp8_recipe='delayed', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.95, adam_eps=1e-15, adam_bias_correction=False, decoupled_weight_decay=True, adamw_lr_mup_scaler=False, sgd_momentum=0.9, adamh_scope='lm_head', adamh_radius=None, adamh_eps=1e-08, spectra_scope='matrices', spectra_hyperball=False, spectra_clip=1.0, spectra_grad_clip=None, spectra_ns_steps=10, muon_momentum=0.9, muon_split_qkv=True, muon_qkv_split_mode='component', muon_split_fc1=True, muon_use_nesterov=False, muon_scale_mode='spectral_mup', muon_fp32_matmul_prec='medium', muon_num_ns_steps=5, muon_tp_mode='blockwise', muon_extra_scale_factor=1.0, muon_split_moe_experts=True, muon_vectorize=[], muon_vectorize_attn_dim='hidden_size', muon_scale_vectorized_mode='full', spectral_ball_momentum=0.9, spectral_ball_use_nesterov=True, spectral_ball_split_qkv=True, spectral_ball_qkv_split_mode='component', spectral_ball_split_fc1=True, spectral_ball_split_moe_experts=True, spectral_ball_msign_steps=8, spectral_ball_solver='bisection', spectral_ball_solver_tolerance_f=1e-08, spectral_ball_solver_max_iterations=20, spectral_ball_radius_mode='spectral_mup', spectral_ball_power_iteration_steps=20, spectral_ball_scale_mode='spectral_mup', spectral_ball_retract_mode='hard', spectral_ball_retract_alpha=0.05, muon_ball_momentum=0.9, muon_ball_use_nesterov=True, muon_ball_split_qkv=True, muon_ball_qkv_split_mode='component', muon_ball_split_fc1=True, muon_ball_split_moe_experts=True, muon_ball_msign_steps=5, muon_ball_radius_mode='spectral_mup', muon_ball_power_iteration_steps=10, muon_ball_scale_mode='spectral_mup', muon_ball_retract_mode='hard', muon_ball_retract_alpha=0.05, use_distributed_optimizer=True, overlap_param_gather=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, log_per_module_update_rms=False, log_per_module_grad_rms=True, barrier_with_L1_time=False, timers=, config_logger_dir='') +setting training iterations to 2384 +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: constant +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints at iteration 6855 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +not setting consumed_train_samples, consumed_valid_samples and skipped_train_samples ... (override_opt_param_scheduler is set) + checkpoint version 3.0 +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer_param_scheduler: > overriding learning rate value to 0.0005 +INFO:megatron.core.optimizer_param_scheduler: > overriding minimum learning rate value to 0.0005 +INFO:megatron.core.optimizer_param_scheduler: > overriding warmup iterations value to 0 +INFO:megatron.core.optimizer_param_scheduler: > overriding total number of iterations value to 76288 +INFO:megatron.core.optimizer_param_scheduler: > overriding learning rate decay style value to constant +INFO:megatron.core.optimizer_param_scheduler: > overriding start weight decay value to 0.1 +INFO:megatron.core.optimizer_param_scheduler: > overriding end weight decay value to 0.1 +INFO:megatron.core.optimizer_param_scheduler: > overriding total number of weight decay iterations value to 76288 +INFO:megatron.core.optimizer_param_scheduler: > overriding weight decay incr style value to constant +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] at iteration 0 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-10-11 02:48:47 +> building train, validation, and test datasets ... + train: 76288 + validation: 320 + test: 320 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 0.999), (0.999, 1.0), None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(76288, 320, 320) and config=GPTDatasetConfig(random_seed=1234, sequence_length=524288, blend=(['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp'], None), blend_per_split=None, multiple_validation_sets=False, full_validation=False, split='999,1,0', split_matrix=[(0, 0.999), (0.999, 1.0), None], num_dataset_builder_threads=1, path_to_cache='/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B', mmap_bin_files=True, fast_cache_load=False, defer_npy_index_mmap=False, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, allow_ambiguous_pad_tokens=False, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=False, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None, load_complemental_dataset=['loss_mask'], document_packing_algorithm='obfd', packing_tokens_per_shard=100000000000, enable_debug_logging=False, emit_document_packing_metadata=True, start_samples=0, context_parallel_size=16, data_parallel_size=4, packing_dp_size=None, sequence_parallel_size=0, hybrid_context_parallel=False) +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + + +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 2031 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 2031 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 15046 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 15046 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 3299 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 3299 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 9228 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 9228 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 166555 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 166555 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 7133 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 7133 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 25714 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 25714 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 86039 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 86039 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 11193 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 11193 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 63250 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 63250 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 36116 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 36116 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 527 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 527 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 59344 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 59344 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 294 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 294 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 484 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 484 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 122883 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 122883 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 163 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 163 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 63156 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 63156 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 778640 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 778640 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 43604 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 43604 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 175670 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 175670 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 226991 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 226991 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 8162 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 8162 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 12434 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 12434 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 28587 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 28587 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 199602 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 199602 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp.idx +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 2784740 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 2784740 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp.idx +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1829337 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1829337 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 178682 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 178682 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 52867 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 52867 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 498 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 498 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 177418 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 177418 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 519 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 519 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 78 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 78 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 84548 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 84548 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 16160 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 16160 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 93319 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 93319 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 12473 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 12473 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 228187 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 228187 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 68576 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 68576 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 189 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 189 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1647592 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1647592 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 89874 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 89874 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 373365 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 373365 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 604 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 604 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 86064 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 86064 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 17995 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 17995 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 151034 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 151034 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1849 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1849 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 148 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 148 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 35370 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 35370 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 274 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 274 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 134522 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 134522 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 79269 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 79269 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 432 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 432 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 145607 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 145607 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 579 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 579 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 31211 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 31211 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 287047 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 287047 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 130201 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 130201 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 70545 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 70545 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 104516 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 104516 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 52569 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 52569 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 413 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 413 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 39658 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 39658 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 19381 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 19381 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 263497 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 263497 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 19431 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 19431 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 69929 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 69929 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 24205 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 24205 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 142217 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 142217 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 70554 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 70554 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 260245 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 260245 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 16637 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 16637 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 10723 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 10723 +>>> start_samples: 0 +INFO:megatron.core.datasets.gpt_dataset: > Found complemental loss_mask dataset. Loading from: /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_loss_mask_tmp +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 2031 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 2031 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 15046 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 15046 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 3299 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 3299 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 9228 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 9228 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 166555 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 166555 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 7133 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 7133 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 25714 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 25714 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 86039 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 86039 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 11193 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 11193 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 63250 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 63250 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 36116 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 36116 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 527 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 527 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 59344 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 59344 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 294 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 294 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 484 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 484 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 122883 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 122883 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 163 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 163 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 63156 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 63156 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 778640 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 778640 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 43604 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 43604 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 175670 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 175670 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 226991 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 226991 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 8162 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 8162 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 12434 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 12434 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 28587 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 28587 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 199602 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 199602 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 2784740 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 2784740 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1829337 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1829337 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 178682 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 178682 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 52867 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 52867 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 498 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 498 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 177418 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 177418 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 519 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 519 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 78 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 78 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 84548 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 84548 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 16160 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 16160 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 93319 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 93319 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 12473 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 12473 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 228187 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 228187 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 68576 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 68576 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 189 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 189 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1647592 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1647592 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 89874 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 89874 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 373365 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 373365 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 604 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 604 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 86064 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 86064 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 17995 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 17995 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 151034 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 151034 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1849 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1849 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 148 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 148 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 35370 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 35370 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 274 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 274 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 134522 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 134522 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 79269 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 79269 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 432 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 432 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 145607 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 145607 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 579 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 579 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 31211 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 31211 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 287047 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 287047 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 130201 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 130201 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 70545 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 70545 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 104516 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 104516 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 52569 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 52569 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 413 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 413 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 39658 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 39658 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 19381 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 19381 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 263497 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 263497 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 19431 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 19431 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 69929 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 69929 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 24205 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 24205 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 142217 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 142217 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 70554 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 70554 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 260245 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 260245 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 16637 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 16637 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 10723 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 10723 +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +INFO:megatron.core.datasets.gpt_dataset:{ + "class": "GPTDataset", + "dataset_path": "/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp", + "num_samples": 76288, + "index_split": "train", + "random_seed": 1234, + "sequence_length": 524288, + "split": "999,1,0", + "split_matrix": [ + [ + 0, + 0.999 + ], + [ + 0.999, + 1.0 + ], + null + ], + "tokenizer": { + "class": "megatron.core.tokenizers.text.models.default_tokenizer.DefaultTokenizerText", + "tokenizer_path": "/mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp", + "vocab_file": "None", + "merges_file": "None" + } +} +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 145599 +>>> start_samples: 0 +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 + +INFO:megatron.core.datasets.gpt_dataset: > Found complemental loss_mask dataset. Loading from: /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_loss_mask_tmp + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + + +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset +Initializing GlobIndexedDataset + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 2031 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 2031 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 15046 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 15046 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 3299 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 3299 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 9228 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 9228 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 166555 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 166555 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 7133 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 7133 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 25714 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 25714 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 86039 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 86039 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 11193 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 11193 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 63250 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 63250 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 36116 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 36116 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 527 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 527 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 59344 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 59344 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 294 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 294 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 484 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 484 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 122883 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 122883 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 163 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 163 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 63156 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 63156 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 778640 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 778640 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 43604 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 43604 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 175670 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 175670 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 226991 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 226991 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 8162 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 8162 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 12434 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 12434 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 28587 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 28587 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 199602 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 199602 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 2784740 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 2784740 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1829337 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1829337 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 178682 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 178682 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 52867 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 52867 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 498 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 498 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 177418 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 177418 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 519 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 519 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 78 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 78 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 84548 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 84548 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 16160 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 16160 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 93319 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 93319 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 12473 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 12473 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 228187 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 228187 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 68576 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 68576 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 189 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 189 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1647592 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1647592 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 89874 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 89874 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 373365 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 373365 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 604 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 604 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 86064 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 86064 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 17995 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 17995 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 151034 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 151034 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1849 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1849 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 148 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 148 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 35370 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 35370 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 274 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 274 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 134522 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 134522 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 79269 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 79269 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 432 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 432 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 145607 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 145607 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 579 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 579 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 31211 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 31211 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 287047 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 287047 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 130201 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 130201 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 70545 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 70545 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 104516 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 104516 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 52569 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 52569 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 413 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 413 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 39658 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 39658 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 19381 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 19381 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 263497 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 263497 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 19431 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 19431 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 69929 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 69929 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 24205 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 24205 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 142217 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 142217 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 70554 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 70554 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 260245 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 260245 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 16637 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 16637 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 10723 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 10723 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +INFO:megatron.core.datasets.gpt_dataset:{ + "class": "GPTDataset", + "dataset_path": "/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp", + "num_samples": 320, + "index_split": "valid", + "random_seed": 1234, + "sequence_length": 524288, + "split": "999,1,0", + "split_matrix": [ + [ + 0, + 0.999 + ], + [ + 0.999, + 1.0 + ], + null + ], + "tokenizer": { + "class": "megatron.core.tokenizers.text.models.default_tokenizer.DefaultTokenizerText", + "tokenizer_path": "/mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp", + "vocab_file": "None", + "merges_file": "None" + } +} +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset valid indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 3495 +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +> finished creating GPT datasets ... GPTDataset +> Building pretraining data loader with total train samples: 145599, consumed train samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDataset +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +[after dataloaders are built] datetime: 2026-10-11 02:49:15 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-10-11 02:49:15 +Iter 0 increase timing-log-level to 2. +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + + + + + +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +WARNING:megatron.core.rerun_state_machine:Result validation enabled +Theoretical memory footprints not yet supported for hybrid Mamba-Transformer models. +[Rank 2] (after 1 iterations) memory (MB) | allocated: 56190.21337890625 | max allocated: 62834.4560546875 | reserved: 66902.0 | max reserved: 66902.0 | device usage: 75139.3125[Rank 6] (after 1 iterations) memory (MB) | allocated: 56190.21337890625 | max allocated: 62852.13427734375 | reserved: 67022.0 | max reserved: 67022.0 | device usage: 75259.3125[Rank 3] (after 1 iterations) memory (MB) | allocated: 56190.21484375 | max allocated: 63105.52880859375 | reserved: 66524.0 | max reserved: 66524.0 | device usage: 74761.3125[Rank 7] (after 1 iterations) memory (MB) | allocated: 56190.21533203125 | max allocated: 62787.921875 | reserved: 67284.0 | max reserved: 67284.0 | device usage: 75313.3125[Rank 0] (after 1 iterations) memory (MB) | allocated: 56190.208984375 | max allocated: 62839.87646484375 | reserved: 66588.0 | max reserved: 66588.0 | device usage: 74719.3125[Rank 5] (after 1 iterations) memory (MB) | allocated: 56190.2060546875 | max allocated: 62796.70361328125 | reserved: 66922.0 | max reserved: 66922.0 | device usage: 75159.3125[Rank 1] (after 1 iterations) memory (MB) | allocated: 56190.2119140625 | max allocated: 62833.15478515625 | reserved: 66494.0 | max reserved: 66494.0 | device usage: 74731.3125 +[Rank 4] (after 1 iterations) memory (MB) | allocated: 56190.20751953125 | max allocated: 63151.6796875 | reserved: 66642.0 | max reserved: 66642.0 | device usage: 74879.3125 + + + + + + +Iter 5 reset timing-log-level. checkpoint path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints; tensorboard path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard +saving checkpoint at iteration 40 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable +W1011 04:40:56.849000 1037778 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 04:40:56.849000 1037778 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( + scheduled an async checkpoint save at iteration 40 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 40 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] +[rank6]: Traceback (most recent call last): +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py", line 907, in +[rank6]: pretrain( +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/training/training.py", line 1212, in pretrain +[rank6]: iteration, num_floating_point_operations_so_far = train( +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/training/training.py", line 4276, in train +[rank6]: ) = train_step( +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/training/training.py", line 2320, in train_step +[rank6]: losses_reduced = forward_backward_func( +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/pipeline_parallel/schedules.py", line 730, in forward_backward_no_pipelining +[rank6]: backward_step( +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/pipeline_parallel/schedules.py", line 552, in backward_step +[rank6]: custom_backward(output_tensor[0], output_tensor_grad[0]) +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/pipeline_parallel/schedules.py", line 211, in custom_backward +[rank6]: Variable._execution_engine.run_backward( +[rank6]: File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/autograd/function.py", line 311, in apply +[rank6]: return user_fn(self, *args) +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/tensor_parallel/random.py", line 468, in backward +[rank6]: outputs = ctx.run_function(*detached_inputs) +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/transformer/transformer_block.py", line 795, in custom_forward +[rank6]: hidden_states, context = layer( +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/transformer/transformer_layer.py", line 1328, in __call__ +[rank6]: return super().__call__(*args, **kwargs) +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/transformer/module.py", line 305, in __call__ +[rank6]: return super().__call__(*args, **kwargs) +[rank6]: File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1879, in _call_impl +[rank6]: return inner() +[rank6]: File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1827, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/transformer/transformer_layer.py", line 507, in forward +[rank6]: output = self._forward_mlp( +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/transformer/transformer_layer.py", line 965, in _forward_mlp +[rank6]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output, **mlp_kwargs) +[rank6]: File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1879, in _call_impl +[rank6]: return inner() +[rank6]: File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1827, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/transformer/moe/moe_layer.py", line 1112, in forward +[rank6]: outputs = custom_forward(hidden_states) +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/transformer/moe/moe_layer.py", line 1095, in custom_forward +[rank6]: return custom_forward_impl(hidden_states) +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/transformer/moe/moe_layer.py", line 1085, in custom_forward_impl +[rank6]: output, mlp_bias = self.routed_experts_compute(dispatched_input, probs, residual) +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/transformer/moe/moe_layer.py", line 666, in routed_experts_compute +[rank6]: output = self.token_dispatcher.combine_preprocess(expert_output) +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/transformer/moe/token_dispatcher.py", line 792, in combine_preprocess +[rank6]: hidden_states, _ = sort_chunks_by_idxs( +[rank6]: File "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/transformer/moe/moe_utils.py", line 613, in sort_chunks_by_idxs +[rank6]: return fused_sort_chunks_by_index(input, split_sizes, sorted_idxs), None +[rank6]: File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformer_engine/pytorch/permutation.py", line 743, in moe_sort_chunks_by_index +[rank6]: output, _ = _moe_chunk_sort.apply(inp, split_sizes, sorted_index, None) +[rank6]: File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/autograd/function.py", line 576, in apply +[rank6]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank6]: File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformer_engine/pytorch/permutation.py", line 661, in forward +[rank6]: output, permuted_probs = triton_permutation.sort_chunks_by_map( +[rank6]: File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformer_engine/pytorch/triton/permutation.py", line 389, in sort_chunks_by_map +[rank6]: output = torch.empty((num_tokens, hidden_size), dtype=inp.dtype, device="cuda") +[rank6]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 1.93 GiB. GPU 6 has a total capacity of 79.25 GiB of which 1.91 GiB is free. Including non-PyTorch memory, this process has 77.31 GiB memory in use. Of the allocated memory 68.54 GiB is allocated by PyTorch, and 1.27 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +[rank6]:[W1011 05:34:18.139860883 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W1011 05:34:23.310000 890102 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 894730 closing signal SIGTERM +W1011 05:34:23.316000 890102 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 894731 closing signal SIGTERM +W1011 05:34:23.320000 890102 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 894732 closing signal SIGTERM +W1011 05:34:23.338000 890102 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 894733 closing signal SIGTERM +W1011 05:34:23.361000 890102 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 894734 closing signal SIGTERM +W1011 05:34:23.385000 890102 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 894735 closing signal SIGTERM +W1011 05:34:23.397000 890102 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 894737 closing signal SIGTERM +W1011 05:34:53.451000 890102 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 894730 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 05:34:56.501000 890102 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 894731 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 05:34:59.527000 890102 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 894732 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 05:35:02.345000 890102 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 894733 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 05:35:05.336000 890102 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 894734 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 05:35:08.114000 890102 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 894735 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 05:35:11.212000 890102 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 894737 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +E1011 05:35:14.735000 890102 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 1) local_rank: 6 (pid: 894736) of binary: /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +Traceback (most recent call last): + File "/mnt/ssd/yulan/cache/installs/cpython-3.10.19-linux-x86_64-gnu/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/mnt/ssd/yulan/cache/installs/cpython-3.10.19-linux-x86_64-gnu/lib/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 905, in + main() + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 143, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 277, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-10-11_05:34:23 + host : cs-517b1-57d54-server + rank : 6 (local_rank: 6) + exitcode : 1 (pid: 894736) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node1.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node1.log new file mode 100644 index 0000000000000000000000000000000000000000..9552a6754e8dbb1e053ba7a1b182d0d98b98cc4b --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node1.log @@ -0,0 +1,1268 @@ +W1011 02:40:41.051000 49478 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:41.051000 49478 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +W1011 02:40:41.062000 49472 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:41.062000 49472 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:41.083000 49477 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:41.083000 49477 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:41.125000 49475 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:41.125000 49475 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:41.166000 49474 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:41.166000 49474 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:41.229000 49473 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:41.229000 49473 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:41.272000 49476 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:41.272000 49476 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:41.324000 49471 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:41.324000 49471 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank 11 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank11]:[W1011 02:41:49.521227816 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank11]:[W1011 02:41:49.521238611 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 13 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 9 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 15 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank13]:[W1011 02:41:49.521438004 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[Gloo] Rank [rank9]:[W1011 02:41:49.521479280 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank13]:[W1011 02:41:49.521491314 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +14 is connected to 63 peer ranks. [rank9]:[W1011 02:41:49.521501304 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +Expected number of connected peer ranks is : 63 +[rank15]:[W1011 02:41:49.521522474 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank15]:[W1011 02:41:49.521532245 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank14]:[W1011 02:41:49.521611884 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank14]:[W1011 02:41:49.521622446 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 8 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank8]:[W1011 02:41:49.521933115 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank8]:[W1011 02:41:49.521943469 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 10 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank10]:[W1011 02:41:50.526021386 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank10]:[W1011 02:41:50.526032231 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 12 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank12]:[W1011 02:41:50.526725888 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank12]:[W1011 02:41:50.526734970 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank [Gloo] Rank 0 is connected to 30 peer ranks. is connected to Expected number of connected peer ranks is : 33 peer ranks. +Expected number of connected peer ranks is : 3 +[Gloo] Rank [Gloo] Rank 0 is connected to 30 peer ranks. is connected to Expected number of connected peer ranks is : 33 peer ranks. +Expected number of connected peer ranks is : 3 +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank9]:[W1011 02:41:52.628494574 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 9] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank15]:[W1011 02:41:52.635253400 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 15] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank8]:[W1011 02:41:52.638342189 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 8] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank11]:[W1011 02:41:52.640607287 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 11] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank13]:[W1011 02:41:52.642313100 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 13] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank10]:[W1011 02:41:52.643806941 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 10] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank14]:[W1011 02:41:52.645507632 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 14] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank12]:[W1011 02:41:52.645886542 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 12] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank8]:[W1011 02:41:52.422934917 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank8]:[W1011 02:41:52.422959073 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank14]:[W1011 02:41:52.422996106 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank14]:[W1011 02:41:52.423015170 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank15]:[W1011 02:41:52.423368104 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank15]:[W1011 02:41:52.423388649 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank10]:[W1011 02:41:52.423919414 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank10]:[W1011 02:41:52.423941141 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank11]:[W1011 02:41:52.423989775 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank12]:[W1011 02:41:52.423993641 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank11]:[W1011 02:41:52.424012957 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank12]:[W1011 02:41:52.424017739 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank9]:[W1011 02:41:52.424021483 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank9]:[W1011 02:41:52.424035860 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank13]:[W1011 02:41:52.424358029 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank13]:[W1011 02:41:52.424375473 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0 + +>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +>>> start_samples: 0Initializing GlobIndexedDataset + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + + + +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + + + + + + + +Initializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDataset +Initializing GlobIndexedDataset + + + +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + + +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +[Rank 9] (after 1 iterations) memory (MB) | allocated: 56190.21728515625 | max allocated: 62863.6962890625 | reserved: 66826.0 | max reserved: 66826.0 | device usage: 75191.5625[Rank 11] (after 1 iterations) memory (MB) | allocated: 56190.2158203125 | max allocated: 62941.2705078125 | reserved: 67202.0 | max reserved: 67202.0 | device usage: 75567.5625[Rank 10] (after 1 iterations) memory (MB) | allocated: 56190.212890625 | max allocated: 62875.45849609375 | reserved: 66962.0 | max reserved: 66962.0 | device usage: 75347.5625[Rank 13] (after 1 iterations) memory (MB) | allocated: 56190.2109375 | max allocated: 63028.78515625 | reserved: 66824.0 | max reserved: 66824.0 | device usage: 75189.5625[Rank 8] (after 1 iterations) memory (MB) | allocated: 56190.20751953125 | max allocated: 62826.5234375 | reserved: 66544.0 | max reserved: 66544.0 | device usage: 74977.5625[Rank 15] (after 1 iterations) memory (MB) | allocated: 56190.2138671875 | max allocated: 62791.09423828125 | reserved: 66810.0 | max reserved: 66810.0 | device usage: 74967.5625[Rank 14] (after 1 iterations) memory (MB) | allocated: 56190.212890625 | max allocated: 62922.13134765625 | reserved: 67148.0 | max reserved: 67148.0 | device usage: 75533.5625 + + + + + + +[Rank 12] (after 1 iterations) memory (MB) | allocated: 56190.2099609375 | max allocated: 63150.31396484375 | reserved: 67128.0 | max reserved: 67128.0 | device usage: 75513.5625 +W1011 04:40:47.798000 18209 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 04:40:47.798000 18209 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank11]:[E1011 07:33:53.411718306 ProcessGroupNCCL.cpp:685] [Rank 11] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=159741, OpType=_ALLGATHER_BASE, NumelIn=262144, NumelOut=4194304, Timeout(ms)=7200000) ran for 7200008 milliseconds before timing out. +[rank11]:[E1011 07:33:53.416786493 ProcessGroupNCCL.cpp:2252] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 11] failure detected by watchdog at work sequence id: 159741 PG status: last enqueued work: 159744, last completed work: 159740 +[rank11]:[E1011 07:33:53.417095024 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank11]:[E1011 07:33:53.418658438 ProcessGroupNCCL.cpp:2584] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 11] First PG on this rank to signal dumping. +[rank9]:[E1011 07:33:53.425473407 ProcessGroupNCCL.cpp:685] [Rank 9] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=159741, OpType=_ALLGATHER_BASE, NumelIn=262144, NumelOut=4194304, Timeout(ms)=7200000) ran for 7200026 milliseconds before timing out. +[rank9]:[E1011 07:33:53.425595535 ProcessGroupNCCL.cpp:2252] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 9] failure detected by watchdog at work sequence id: 159741 PG status: last enqueued work: 159744, last completed work: 159740 +[rank9]:[E1011 07:33:53.425609308 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank9]:[E1011 07:33:53.425694081 ProcessGroupNCCL.cpp:2584] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 9] First PG on this rank to signal dumping. +[rank8]:[E1011 07:33:53.428916692 ProcessGroupNCCL.cpp:685] [Rank 0] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200021 milliseconds before timing out. +[rank8]:[E1011 07:33:53.429027061 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] failure detected by watchdog at work sequence id: 52272 PG status: last enqueued work: 52272, last completed work: 52271 +[rank8]:[E1011 07:33:53.429042827 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank8]:[E1011 07:33:53.429122626 ProcessGroupNCCL.cpp:2584] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] First PG on this rank to signal dumping. +[rank12]:[E1011 07:33:53.433623020 ProcessGroupNCCL.cpp:685] [Rank 4] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200026 milliseconds before timing out. +[rank12]:[E1011 07:33:53.433727035 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] failure detected by watchdog at work sequence id: 52272 PG status: last enqueued work: 52272, last completed work: 52271 +[rank12]:[E1011 07:33:53.433739756 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank12]:[E1011 07:33:53.433807927 ProcessGroupNCCL.cpp:2584] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] First PG on this rank to signal dumping. +[rank10]:[E1011 07:33:53.437169078 ProcessGroupNCCL.cpp:685] [Rank 10] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=159741, OpType=_ALLGATHER_BASE, NumelIn=262144, NumelOut=4194304, Timeout(ms)=7200000) ran for 7200038 milliseconds before timing out. +[rank10]:[E1011 07:33:53.437325315 ProcessGroupNCCL.cpp:2252] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 10] failure detected by watchdog at work sequence id: 159741 PG status: last enqueued work: 159744, last completed work: 159740 +[rank10]:[E1011 07:33:53.437339950 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank10]:[E1011 07:33:53.437409241 ProcessGroupNCCL.cpp:2584] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 10] First PG on this rank to signal dumping. +[rank10]:[E1011 07:33:53.441953611 ProcessGroupNCCL.cpp:685] [Rank 10] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200035 milliseconds before timing out. +[rank10]:[E1011 07:33:53.441991663 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] failure detected by watchdog at work sequence id: 52272 PG status: last enqueued work: 52272, last completed work: 52271 +[rank10]:[E1011 07:33:53.441999117 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank13]:[E1011 07:33:53.442462584 ProcessGroupNCCL.cpp:685] [Rank 5] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200035 milliseconds before timing out. +[rank13]:[E1011 07:33:53.442575437 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] failure detected by watchdog at work sequence id: 52272 PG status: last enqueued work: 52272, last completed work: 52271 +[rank13]:[E1011 07:33:53.442595613 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank13]:[E1011 07:33:53.442673825 ProcessGroupNCCL.cpp:2584] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] First PG on this rank to signal dumping. +[rank15]:[E1011 07:33:53.443095871 ProcessGroupNCCL.cpp:685] [Rank 15] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=159741, OpType=_ALLGATHER_BASE, NumelIn=262144, NumelOut=4194304, Timeout(ms)=7200000) ran for 7200044 milliseconds before timing out. +[rank15]:[E1011 07:33:53.443194701 ProcessGroupNCCL.cpp:2252] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 15] failure detected by watchdog at work sequence id: 159741 PG status: last enqueued work: 159744, last completed work: 159740 +[rank15]:[E1011 07:33:53.443208220 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank15]:[E1011 07:33:53.443281891 ProcessGroupNCCL.cpp:2584] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 15] First PG on this rank to signal dumping. +[rank14]:[E1011 07:33:53.466158079 ProcessGroupNCCL.cpp:685] [Rank 14] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=159741, OpType=_ALLGATHER_BASE, NumelIn=262144, NumelOut=4194304, Timeout(ms)=7200000) ran for 7200067 milliseconds before timing out. +[rank14]:[E1011 07:33:53.466264156 ProcessGroupNCCL.cpp:2252] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 14] failure detected by watchdog at work sequence id: 159741 PG status: last enqueued work: 159744, last completed work: 159740 +[rank14]:[E1011 07:33:53.466278614 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank14]:[E1011 07:33:53.466353401 ProcessGroupNCCL.cpp:2584] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 14] First PG on this rank to signal dumping. +[rank13]:[E1011 07:33:53.467864915 ProcessGroupNCCL.cpp:685] [Rank 5] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=159741, OpType=_ALLGATHER_BASE, NumelIn=262144, NumelOut=4194304, Timeout(ms)=7200000) ran for 7200069 milliseconds before timing out. +[rank13]:[E1011 07:33:53.467905927 ProcessGroupNCCL.cpp:2252] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 13] failure detected by watchdog at work sequence id: 159741 PG status: last enqueued work: 159744, last completed work: 159740 +[rank13]:[E1011 07:33:53.467914126 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank14]:[E1011 07:33:53.469357896 ProcessGroupNCCL.cpp:685] [Rank 14] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200062 milliseconds before timing out. +[rank14]:[E1011 07:33:53.469393313 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] failure detected by watchdog at work sequence id: 52272 PG status: last enqueued work: 52272, last completed work: 52271 +[rank14]:[E1011 07:33:53.469402938 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank12]:[E1011 07:33:53.472632932 ProcessGroupNCCL.cpp:685] [Rank 4] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=159741, OpType=_ALLGATHER_BASE, NumelIn=262144, NumelOut=4194304, Timeout(ms)=7200000) ran for 7200074 milliseconds before timing out. +[rank12]:[E1011 07:33:53.472675535 ProcessGroupNCCL.cpp:2252] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 12] failure detected by watchdog at work sequence id: 159741 PG status: last enqueued work: 159744, last completed work: 159740 +[rank12]:[E1011 07:33:53.472683468 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank8]:[E1011 07:33:53.479454554 ProcessGroupNCCL.cpp:685] [Rank 0] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=159741, OpType=_ALLGATHER_BASE, NumelIn=262144, NumelOut=4194304, Timeout(ms)=7200000) ran for 7200080 milliseconds before timing out. +[rank8]:[E1011 07:33:53.479494925 ProcessGroupNCCL.cpp:2252] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 8] failure detected by watchdog at work sequence id: 159741 PG status: last enqueued work: 159744, last completed work: 159740 +[rank8]:[E1011 07:33:53.479502311 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank11]:[E1011 07:33:53.482044673 ProcessGroupNCCL.cpp:685] [Rank 11] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200075 milliseconds before timing out. +[rank11]:[E1011 07:33:53.482083997 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] failure detected by watchdog at work sequence id: 52272 PG status: last enqueued work: 52272, last completed work: 52271 +[rank11]:[E1011 07:33:53.482094154 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank15]:[E1011 07:33:53.483884379 ProcessGroupNCCL.cpp:685] [Rank 15] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200076 milliseconds before timing out. +[rank9]:[E1011 07:33:53.483897345 ProcessGroupNCCL.cpp:685] [Rank 9] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200077 milliseconds before timing out. +[rank15]:[E1011 07:33:53.483927705 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] failure detected by watchdog at work sequence id: 52272 PG status: last enqueued work: 52272, last completed work: 52271 +[rank15]:[E1011 07:33:53.483947895 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank9]:[E1011 07:33:53.483966889 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] failure detected by watchdog at work sequence id: 52272 PG status: last enqueued work: 52272, last completed work: 52271 +[rank9]:[E1011 07:33:53.483976809 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank14]:[W1011 07:33:53.469474395 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=98, addr=[cs-bca8a-46a8d-server]:43256, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f7c3317eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f7c171bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f7c171bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f7c171bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f7c171b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f7bd6695880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f7bd669599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f7bd66a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f7bd66a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f7bba001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f7c3440baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f7c34498c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank11]:[W1011 07:33:53.482160246 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=98, addr=[cs-bca8a-46a8d-server]:43244, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f0df16d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f0dd57bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f0dd57bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f0dd57bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f0dd57b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f0d94c95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f0d94c9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f0d94ca2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f0d94ca4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f0d78601db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f0df28ffaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f0df298cc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank12]:[W1011 07:33:53.472756022 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=98, addr=[cs-bca8a-46a8d-server]:43230, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f4e705deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f4eb1fbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f4eb1fbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f4eb1fbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f4eb1fb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f4e71495880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f4e7149599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f4e714a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f4e714a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f4e54c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f4ecf057aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f4ecf0e4c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank8]:[W1011 07:33:53.479571131 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=98, addr=[cs-bca8a-46a8d-server]:43252, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fd9c9cd9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fd9addbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fd9addbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fd9addbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fd9addb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fd96d295880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fd96d29599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fd96d2a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fd96d2a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fd950c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fd9caf42aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fd9cafcfc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank10]:[W1011 07:33:53.445308246 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=98, addr=[cs-bca8a-46a8d-server]:43232, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f62925deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f62d3fbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f62d3fbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f62d3fbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f62d3fb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f6293495880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f629349599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f62934a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f62934a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f6276c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f62f105baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f62f10e8c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank13]:[W1011 07:33:53.467989679 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=98, addr=[cs-bca8a-46a8d-server]:43272, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f9fc40d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f9fa81bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f9fa81bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f9fa81bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f9fa81b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f9f67695880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f9f6769599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f9f676a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f9f676a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f9f4b001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f9fc5344aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f9fc53d1c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank9]:[W1011 07:33:53.484046966 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=98, addr=[cs-bca8a-46a8d-server]:43228, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbd8c37eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fbd703bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fbd703bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fbd703bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fbd703b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fbd2f895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fbd2f89599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fbd2f8a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fbd2f8a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fbd13201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fbd8d581aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fbd8d60ec3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank15]:[W1011 07:33:53.484039981 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=98, addr=[cs-bca8a-46a8d-server]:43226, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f6af6f7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f6adafbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f6adafbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f6adafbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f6adafb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f6a9a495880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f6a9a49599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f6a9a4a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f6a9a4a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f6a7de01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f6af8156aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f6af81e3c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank14]:[E1011 07:33:53.489146258 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f7c3317eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f7c171bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f7c171bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f7c171bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f7c171b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f7bd6695880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f7bd669599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f7bd66a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f7bd66a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f7bba001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f7c3440baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f7c34498c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank12]:[E1011 07:33:53.489154647 ProcessGroupNCCL.cpp:2574] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 12] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f4e705deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f4eb1fbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f4eb1fbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f4eb1fbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f4eb1fb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f4e71495880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f4e7149599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f4e714a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f4e714a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f4e54c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f4ecf057aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f4ecf0e4c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank15]:[E1011 07:33:53.489153593 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f6af6f7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f6adafbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f6adafbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f6adafbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f6adafb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f6a9a495880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f6a9a49599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f6a9a4a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f6a9a4a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f6a7de01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f6af8156aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f6af81e3c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank8]:[E1011 07:33:53.489158240 ProcessGroupNCCL.cpp:2574] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 8] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fd9c9cd9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fd9addbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fd9addbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fd9addbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fd9addb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fd96d295880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fd96d29599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fd96d2a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fd96d2a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fd950c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fd9caf42aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fd9cafcfc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank9]:[E1011 07:33:53.489162771 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbd8c37eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fbd703bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fbd703bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fbd703bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fbd703b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fbd2f895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fbd2f89599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fbd2f8a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fbd2f8a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fbd13201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fbd8d581aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fbd8d60ec3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank11]:[E1011 07:33:53.489165742 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f0df16d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f0dd57bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f0dd57bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f0dd57bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f0dd57b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f0d94c95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f0d94c9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f0d94ca2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f0d94ca4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f0d78601db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f0df28ffaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f0df298cc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank10]:[E1011 07:33:53.489180868 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f62925deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f62d3fbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f62d3fbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f62d3fbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f62d3fb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f6293495880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f629349599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f62934a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f62934a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f6276c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f62f105baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f62f10e8c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank13]:[E1011 07:33:53.489183292 ProcessGroupNCCL.cpp:2574] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 13] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f9fc40d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f9fa81bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f9fa81bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f9fa81bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f9fa81b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f9f67695880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f9f6769599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f9f676a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f9f676a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f9f4b001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f9fc5344aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f9fc53d1c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank11]:[E1011 07:44:53.575976373 ProcessGroupNCCL.cpp:685] [Rank 11] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=235280, OpType=ALLTOALL_BASE, NumelIn=268451840, NumelOut=215621632, Timeout(ms)=600000) ran for 600010 milliseconds before timing out. +[rank11]:[E1011 07:44:53.576127582 ProcessGroupNCCL.cpp:2252] [PG ID 12 PG GUID 381(EXPERT_MODEL_PARALLEL_GROUP) Rank 3] failure detected by watchdog at work sequence id: 235280 PG status: last enqueued work: 235280, last completed work: 235279 +[rank11]:[E1011 07:44:53.576139335 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank11]:[W1011 07:44:53.576194639 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=98, addr=[cs-bca8a-46a8d-server]:43244, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f0df16d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f0dd57bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f0dd57bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f0dd57bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f0dd57b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f0d94c95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f0d94c9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f0d94ca2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f0d94ca4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f0d78601db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f0df28ffaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f0df298cc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank11]:[E1011 07:44:53.580175215 ProcessGroupNCCL.cpp:2574] [PG ID 12 PG GUID 381(EXPERT_MODEL_PARALLEL_GROUP) Rank 3] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f0df16d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f0dd57bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f0dd57bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f0dd57bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f0dd57b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f0d94c95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f0d94c9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f0d94ca2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f0d94ca4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f0d78601db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f0df28ffaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f0df298cc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank15]:[E1011 07:44:53.586612270 ProcessGroupNCCL.cpp:685] [Rank 15] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=235280, OpType=ALLTOALL_BASE, NumelIn=268451840, NumelOut=328159232, Timeout(ms)=600000) ran for 600021 milliseconds before timing out. +[rank15]:[E1011 07:44:53.586680468 ProcessGroupNCCL.cpp:2252] [PG ID 12 PG GUID 381(EXPERT_MODEL_PARALLEL_GROUP) Rank 7] failure detected by watchdog at work sequence id: 235280 PG status: last enqueued work: 235280, last completed work: 235279 +[rank15]:[E1011 07:44:53.586689760 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank9]:[E1011 07:44:53.586967788 ProcessGroupNCCL.cpp:685] [Rank 9] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=235280, OpType=ALLTOALL_BASE, NumelIn=268451840, NumelOut=291737600, Timeout(ms)=600000) ran for 600021 milliseconds before timing out. +[rank9]:[E1011 07:44:53.587029705 ProcessGroupNCCL.cpp:2252] [PG ID 12 PG GUID 381(EXPERT_MODEL_PARALLEL_GROUP) Rank 1] failure detected by watchdog at work sequence id: 235280 PG status: last enqueued work: 235280, last completed work: 235279 +[rank9]:[E1011 07:44:53.587040708 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank15]:[W1011 07:44:53.586726243 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=98, addr=[cs-bca8a-46a8d-server]:43226, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f6af6f7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f6adafbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f6adafbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f6adafbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f6adafb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f6a9a495880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f6a9a49599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f6a9a4a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f6a9a4a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f6a7de01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f6af8156aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f6af81e3c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank15]:[E1011 07:44:53.590734230 ProcessGroupNCCL.cpp:2574] [PG ID 12 PG GUID 381(EXPERT_MODEL_PARALLEL_GROUP) Rank 7] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f6af6f7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f6adafbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f6adafbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f6adafbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f6adafb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f6a9a495880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f6a9a49599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f6a9a4a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f6a9a4a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f6a7de01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f6af8156aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f6af81e3c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank9]:[W1011 07:44:53.587075193 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=98, addr=[cs-bca8a-46a8d-server]:43228, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbd8c37eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fbd703bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fbd703bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fbd703bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fbd703b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fbd2f895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fbd2f89599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fbd2f8a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fbd2f8a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fbd13201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fbd8d581aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fbd8d60ec3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank9]:[E1011 07:44:53.590926388 ProcessGroupNCCL.cpp:2574] [PG ID 12 PG GUID 381(EXPERT_MODEL_PARALLEL_GROUP) Rank 1] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbd8c37eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fbd703bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fbd703bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fbd703bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fbd703b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fbd2f895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fbd2f89599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fbd2f8a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fbd2f8a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fbd13201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fbd8d581aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fbd8d60ec3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank13]:[E1011 07:44:53.595446499 ProcessGroupNCCL.cpp:685] [Rank 5] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=235280, OpType=ALLTOALL_BASE, NumelIn=268451840, NumelOut=297422848, Timeout(ms)=600000) ran for 600030 milliseconds before timing out. +[rank13]:[E1011 07:44:53.595502204 ProcessGroupNCCL.cpp:2252] [PG ID 12 PG GUID 381(EXPERT_MODEL_PARALLEL_GROUP) Rank 5] failure detected by watchdog at work sequence id: 235280 PG status: last enqueued work: 235280, last completed work: 235279 +[rank13]:[E1011 07:44:53.595510838 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank13]:[W1011 07:44:53.595544837 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=98, addr=[cs-bca8a-46a8d-server]:43272, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f9fc40d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f9fa81bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f9fa81bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f9fa81bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f9fa81b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f9f67695880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f9f6769599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f9f676a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f9f676a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f9f4b001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f9fc5344aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f9fc53d1c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank13]:[E1011 07:44:54.599436375 ProcessGroupNCCL.cpp:2574] [PG ID 12 PG GUID 381(EXPERT_MODEL_PARALLEL_GROUP) Rank 5] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f9fc40d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f9fa81bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f9fa81bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f9fa81bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f9fa81b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f9f67695880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f9f6769599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f9f676a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f9f676a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f9f4b001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f9fc5344aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f9fc53d1c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank14]:[E1011 07:44:54.623316660 ProcessGroupNCCL.cpp:685] [Rank 14] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=235280, OpType=ALLTOALL_BASE, NumelIn=268451840, NumelOut=301780992, Timeout(ms)=600000) ran for 600058 milliseconds before timing out. +[rank14]:[E1011 07:44:54.623370462 ProcessGroupNCCL.cpp:2252] [PG ID 12 PG GUID 381(EXPERT_MODEL_PARALLEL_GROUP) Rank 6] failure detected by watchdog at work sequence id: 235280 PG status: last enqueued work: 235280, last completed work: 235279 +[rank14]:[E1011 07:44:54.623380541 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank14]:[W1011 07:44:54.623415738 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=98, addr=[cs-bca8a-46a8d-server]:43256, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f7c3317eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f7c171bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f7c171bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f7c171bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f7c171b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f7bd6695880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f7bd669599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f7bd66a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f7bd66a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f7bba001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f7c3440baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f7c34498c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank14]:[E1011 07:44:54.627239903 ProcessGroupNCCL.cpp:2574] [PG ID 12 PG GUID 381(EXPERT_MODEL_PARALLEL_GROUP) Rank 6] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f7c3317eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f7c171bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f7c171bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f7c171bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f7c171b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f7bd6695880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f7bd669599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f7bd66a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f7bd66a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f7bba001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f7c3440baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f7c34498c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank10]:[E1011 07:44:54.651469430 ProcessGroupNCCL.cpp:685] [Rank 10] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=235280, OpType=ALLTOALL_BASE, NumelIn=268451840, NumelOut=220231680, Timeout(ms)=600000) ran for 600086 milliseconds before timing out. +[rank10]:[E1011 07:44:54.651520910 ProcessGroupNCCL.cpp:2252] [PG ID 12 PG GUID 381(EXPERT_MODEL_PARALLEL_GROUP) Rank 2] failure detected by watchdog at work sequence id: 235280 PG status: last enqueued work: 235280, last completed work: 235279 +[rank10]:[E1011 07:44:54.651530222 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank8]:[E1011 07:44:54.654449852 ProcessGroupNCCL.cpp:685] [Rank 0] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=235280, OpType=ALLTOALL_BASE, NumelIn=268451840, NumelOut=236539904, Timeout(ms)=600000) ran for 600089 milliseconds before timing out. +[rank8]:[E1011 07:44:54.654504104 ProcessGroupNCCL.cpp:2252] [PG ID 12 PG GUID 381(EXPERT_MODEL_PARALLEL_GROUP) Rank 0] failure detected by watchdog at work sequence id: 235280 PG status: last enqueued work: 235280, last completed work: 235279 +[rank8]:[E1011 07:44:54.654546760 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank12]:[E1011 07:44:54.655125964 ProcessGroupNCCL.cpp:685] [Rank 4] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=235280, OpType=ALLTOALL_BASE, NumelIn=268451840, NumelOut=231479296, Timeout(ms)=600000) ran for 600070 milliseconds before timing out. +[rank12]:[E1011 07:44:54.655184173 ProcessGroupNCCL.cpp:2252] [PG ID 12 PG GUID 381(EXPERT_MODEL_PARALLEL_GROUP) Rank 4] failure detected by watchdog at work sequence id: 235280 PG status: last enqueued work: 235281, last completed work: 235279 +[rank12]:[E1011 07:44:54.655193229 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank10]:[W1011 07:44:54.651561146 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=98, addr=[cs-bca8a-46a8d-server]:43232, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f62925deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f62d3fbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f62d3fbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f62d3fbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f62d3fb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f6293495880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f629349599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f62934a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f62934a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f6276c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f62f105baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f62f10e8c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank10]:[E1011 07:44:54.655447466 ProcessGroupNCCL.cpp:2574] [PG ID 12 PG GUID 381(EXPERT_MODEL_PARALLEL_GROUP) Rank 2] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f62925deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f62d3fbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f62d3fbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f62d3fbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f62d3fb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f6293495880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f629349599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f62934a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f62934a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f6276c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f62f105baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f62f10e8c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank8]:[W1011 07:44:54.654583843 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=98, addr=[cs-bca8a-46a8d-server]:43252, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fd9c9cd9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fd9addbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fd9addbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fd9addbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fd9addb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fd96d295880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fd96d29599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fd96d2a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fd96d2a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fd950c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fd9caf42aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fd9cafcfc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank8]:[E1011 07:44:54.658459089 ProcessGroupNCCL.cpp:2574] [PG ID 12 PG GUID 381(EXPERT_MODEL_PARALLEL_GROUP) Rank 0] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fd9c9cd9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fd9addbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fd9addbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fd9addbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fd9addb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fd96d295880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fd96d29599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fd96d2a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fd96d2a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fd950c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fd9caf42aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fd9cafcfc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank12]:[W1011 07:44:54.655225329 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=98, addr=[cs-bca8a-46a8d-server]:43230, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f4e705deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f4eb1fbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f4eb1fbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f4eb1fbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f4eb1fb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f4e71495880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f4e7149599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f4e714a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f4e714a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f4e54c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f4ecf057aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f4ecf0e4c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank12]:[E1011 07:44:54.659069312 ProcessGroupNCCL.cpp:2574] [PG ID 12 PG GUID 381(EXPERT_MODEL_PARALLEL_GROUP) Rank 4] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f4e705deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f4eb1fbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f4eb1fbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f4eb1fbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f4eb1fb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f4e71495880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f4e7149599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f4e714a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f4e714a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f4e54c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f4ecf057aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f4ecf0e4c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank10]:[E1011 07:45:51.611985461 ProcessGroupNCCL.cpp:1870] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 10] ProcessGroupNCCL's watchdog got stuck for 480 seconds without making progress in monitoring enqueued collectives. This typically indicates a NCCL/CUDA API (e.g., CudaEventDestroy) hang blocking the watchdog, and could be triggered by another thread holding the GIL inside a CUDA api (for example, CudaEventDestroy), or other deadlock-prone behaviors.If you suspect the watchdog is not actually stuck and a longer timeout would help, you can either increase the timeout (TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC) to a larger value or disable the heartbeat monitor (TORCH_NCCL_ENABLE_MONITORING=0).If either of aforementioned helps, feel free to file an issue to PyTorch about the short timeout or false positive abort; otherwise, please attempt to debug the hang. +[rank12]:[E1011 07:45:51.612101219 ProcessGroupNCCL.cpp:1870] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 12] ProcessGroupNCCL's watchdog got stuck for 480 seconds without making progress in monitoring enqueued collectives. This typically indicates a NCCL/CUDA API (e.g., CudaEventDestroy) hang blocking the watchdog, and could be triggered by another thread holding the GIL inside a CUDA api (for example, CudaEventDestroy), or other deadlock-prone behaviors.If you suspect the watchdog is not actually stuck and a longer timeout would help, you can either increase the timeout (TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC) to a larger value or disable the heartbeat monitor (TORCH_NCCL_ENABLE_MONITORING=0).If either of aforementioned helps, feel free to file an issue to PyTorch about the short timeout or false positive abort; otherwise, please attempt to debug the hang. +[rank9]:[E1011 07:45:51.612175083 ProcessGroupNCCL.cpp:1870] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 9] ProcessGroupNCCL's watchdog got stuck for 480 seconds without making progress in monitoring enqueued collectives. This typically indicates a NCCL/CUDA API (e.g., CudaEventDestroy) hang blocking the watchdog, and could be triggered by another thread holding the GIL inside a CUDA api (for example, CudaEventDestroy), or other deadlock-prone behaviors.If you suspect the watchdog is not actually stuck and a longer timeout would help, you can either increase the timeout (TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC) to a larger value or disable the heartbeat monitor (TORCH_NCCL_ENABLE_MONITORING=0).If either of aforementioned helps, feel free to file an issue to PyTorch about the short timeout or false positive abort; otherwise, please attempt to debug the hang. +[rank14]:[E1011 07:45:51.612210658 ProcessGroupNCCL.cpp:1870] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 14] ProcessGroupNCCL's watchdog got stuck for 480 seconds without making progress in monitoring enqueued collectives. This typically indicates a NCCL/CUDA API (e.g., CudaEventDestroy) hang blocking the watchdog, and could be triggered by another thread holding the GIL inside a CUDA api (for example, CudaEventDestroy), or other deadlock-prone behaviors.If you suspect the watchdog is not actually stuck and a longer timeout would help, you can either increase the timeout (TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC) to a larger value or disable the heartbeat monitor (TORCH_NCCL_ENABLE_MONITORING=0).If either of aforementioned helps, feel free to file an issue to PyTorch about the short timeout or false positive abort; otherwise, please attempt to debug the hang. +[rank15]:[E1011 07:45:51.612392989 ProcessGroupNCCL.cpp:1870] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 15] ProcessGroupNCCL's watchdog got stuck for 480 seconds without making progress in monitoring enqueued collectives. This typically indicates a NCCL/CUDA API (e.g., CudaEventDestroy) hang blocking the watchdog, and could be triggered by another thread holding the GIL inside a CUDA api (for example, CudaEventDestroy), or other deadlock-prone behaviors.If you suspect the watchdog is not actually stuck and a longer timeout would help, you can either increase the timeout (TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC) to a larger value or disable the heartbeat monitor (TORCH_NCCL_ENABLE_MONITORING=0).If either of aforementioned helps, feel free to file an issue to PyTorch about the short timeout or false positive abort; otherwise, please attempt to debug the hang. +[rank13]:[E1011 07:45:51.612438471 ProcessGroupNCCL.cpp:1870] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 13] ProcessGroupNCCL's watchdog got stuck for 480 seconds without making progress in monitoring enqueued collectives. This typically indicates a NCCL/CUDA API (e.g., CudaEventDestroy) hang blocking the watchdog, and could be triggered by another thread holding the GIL inside a CUDA api (for example, CudaEventDestroy), or other deadlock-prone behaviors.If you suspect the watchdog is not actually stuck and a longer timeout would help, you can either increase the timeout (TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC) to a larger value or disable the heartbeat monitor (TORCH_NCCL_ENABLE_MONITORING=0).If either of aforementioned helps, feel free to file an issue to PyTorch about the short timeout or false positive abort; otherwise, please attempt to debug the hang. +[rank11]:[E1011 07:45:51.612451873 ProcessGroupNCCL.cpp:1870] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 11] ProcessGroupNCCL's watchdog got stuck for 480 seconds without making progress in monitoring enqueued collectives. This typically indicates a NCCL/CUDA API (e.g., CudaEventDestroy) hang blocking the watchdog, and could be triggered by another thread holding the GIL inside a CUDA api (for example, CudaEventDestroy), or other deadlock-prone behaviors.If you suspect the watchdog is not actually stuck and a longer timeout would help, you can either increase the timeout (TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC) to a larger value or disable the heartbeat monitor (TORCH_NCCL_ENABLE_MONITORING=0).If either of aforementioned helps, feel free to file an issue to PyTorch about the short timeout or false positive abort; otherwise, please attempt to debug the hang. +[rank8]:[E1011 07:45:51.612706481 ProcessGroupNCCL.cpp:1870] [PG ID 3 PG GUID 35(CONTEXT_PARALLEL_GROUP) Rank 8] ProcessGroupNCCL's watchdog got stuck for 480 seconds without making progress in monitoring enqueued collectives. This typically indicates a NCCL/CUDA API (e.g., CudaEventDestroy) hang blocking the watchdog, and could be triggered by another thread holding the GIL inside a CUDA api (for example, CudaEventDestroy), or other deadlock-prone behaviors.If you suspect the watchdog is not actually stuck and a longer timeout would help, you can either increase the timeout (TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC) to a larger value or disable the heartbeat monitor (TORCH_NCCL_ENABLE_MONITORING=0).If either of aforementioned helps, feel free to file an issue to PyTorch about the short timeout or false positive abort; otherwise, please attempt to debug the hang. +[rank15]:[E1011 07:45:51.621537753 ProcessGroupNCCL.cpp:1870] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] ProcessGroupNCCL's watchdog got stuck for 480 seconds without making progress in monitoring enqueued collectives. This typically indicates a NCCL/CUDA API (e.g., CudaEventDestroy) hang blocking the watchdog, and could be triggered by another thread holding the GIL inside a CUDA api (for example, CudaEventDestroy), or other deadlock-prone behaviors.If you suspect the watchdog is not actually stuck and a longer timeout would help, you can either increase the timeout (TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC) to a larger value or disable the heartbeat monitor (TORCH_NCCL_ENABLE_MONITORING=0).If either of aforementioned helps, feel free to file an issue to PyTorch about the short timeout or false positive abort; otherwise, please attempt to debug the hang. +[rank10]:[E1011 07:45:51.621583056 ProcessGroupNCCL.cpp:1870] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] ProcessGroupNCCL's watchdog got stuck for 480 seconds without making progress in monitoring enqueued collectives. This typically indicates a NCCL/CUDA API (e.g., CudaEventDestroy) hang blocking the watchdog, and could be triggered by another thread holding the GIL inside a CUDA api (for example, CudaEventDestroy), or other deadlock-prone behaviors.If you suspect the watchdog is not actually stuck and a longer timeout would help, you can either increase the timeout (TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC) to a larger value or disable the heartbeat monitor (TORCH_NCCL_ENABLE_MONITORING=0).If either of aforementioned helps, feel free to file an issue to PyTorch about the short timeout or false positive abort; otherwise, please attempt to debug the hang. +[rank11]:[E1011 07:45:51.621622966 ProcessGroupNCCL.cpp:1870] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] ProcessGroupNCCL's watchdog got stuck for 480 seconds without making progress in monitoring enqueued collectives. This typically indicates a NCCL/CUDA API (e.g., CudaEventDestroy) hang blocking the watchdog, and could be triggered by another thread holding the GIL inside a CUDA api (for example, CudaEventDestroy), or other deadlock-prone behaviors.If you suspect the watchdog is not actually stuck and a longer timeout would help, you can either increase the timeout (TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC) to a larger value or disable the heartbeat monitor (TORCH_NCCL_ENABLE_MONITORING=0).If either of aforementioned helps, feel free to file an issue to PyTorch about the short timeout or false positive abort; otherwise, please attempt to debug the hang. +[rank12]:[E1011 07:45:51.621786530 ProcessGroupNCCL.cpp:1870] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] ProcessGroupNCCL's watchdog got stuck for 480 seconds without making progress in monitoring enqueued collectives. This typically indicates a NCCL/CUDA API (e.g., CudaEventDestroy) hang blocking the watchdog, and could be triggered by another thread holding the GIL inside a CUDA api (for example, CudaEventDestroy), or other deadlock-prone behaviors.If you suspect the watchdog is not actually stuck and a longer timeout would help, you can either increase the timeout (TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC) to a larger value or disable the heartbeat monitor (TORCH_NCCL_ENABLE_MONITORING=0).If either of aforementioned helps, feel free to file an issue to PyTorch about the short timeout or false positive abort; otherwise, please attempt to debug the hang. +[rank9]:[E1011 07:45:51.621827605 ProcessGroupNCCL.cpp:1870] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] ProcessGroupNCCL's watchdog got stuck for 480 seconds without making progress in monitoring enqueued collectives. This typically indicates a NCCL/CUDA API (e.g., CudaEventDestroy) hang blocking the watchdog, and could be triggered by another thread holding the GIL inside a CUDA api (for example, CudaEventDestroy), or other deadlock-prone behaviors.If you suspect the watchdog is not actually stuck and a longer timeout would help, you can either increase the timeout (TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC) to a larger value or disable the heartbeat monitor (TORCH_NCCL_ENABLE_MONITORING=0).If either of aforementioned helps, feel free to file an issue to PyTorch about the short timeout or false positive abort; otherwise, please attempt to debug the hang. +[rank13]:[E1011 07:45:51.622506404 ProcessGroupNCCL.cpp:1870] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] ProcessGroupNCCL's watchdog got stuck for 480 seconds without making progress in monitoring enqueued collectives. This typically indicates a NCCL/CUDA API (e.g., CudaEventDestroy) hang blocking the watchdog, and could be triggered by another thread holding the GIL inside a CUDA api (for example, CudaEventDestroy), or other deadlock-prone behaviors.If you suspect the watchdog is not actually stuck and a longer timeout would help, you can either increase the timeout (TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC) to a larger value or disable the heartbeat monitor (TORCH_NCCL_ENABLE_MONITORING=0).If either of aforementioned helps, feel free to file an issue to PyTorch about the short timeout or false positive abort; otherwise, please attempt to debug the hang. +[rank8]:[E1011 07:45:51.622586202 ProcessGroupNCCL.cpp:1870] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] ProcessGroupNCCL's watchdog got stuck for 480 seconds without making progress in monitoring enqueued collectives. This typically indicates a NCCL/CUDA API (e.g., CudaEventDestroy) hang blocking the watchdog, and could be triggered by another thread holding the GIL inside a CUDA api (for example, CudaEventDestroy), or other deadlock-prone behaviors.If you suspect the watchdog is not actually stuck and a longer timeout would help, you can either increase the timeout (TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC) to a larger value or disable the heartbeat monitor (TORCH_NCCL_ENABLE_MONITORING=0).If either of aforementioned helps, feel free to file an issue to PyTorch about the short timeout or false positive abort; otherwise, please attempt to debug the hang. +[rank14]:[E1011 07:45:51.624644826 ProcessGroupNCCL.cpp:1870] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] ProcessGroupNCCL's watchdog got stuck for 480 seconds without making progress in monitoring enqueued collectives. This typically indicates a NCCL/CUDA API (e.g., CudaEventDestroy) hang blocking the watchdog, and could be triggered by another thread holding the GIL inside a CUDA api (for example, CudaEventDestroy), or other deadlock-prone behaviors.If you suspect the watchdog is not actually stuck and a longer timeout would help, you can either increase the timeout (TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC) to a larger value or disable the heartbeat monitor (TORCH_NCCL_ENABLE_MONITORING=0).If either of aforementioned helps, feel free to file an issue to PyTorch about the short timeout or false positive abort; otherwise, please attempt to debug the hang. +[rank11]:[E1011 07:45:53.592704219 ProcessGroupNCCL.cpp:746] [Rank 11] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank11]:[E1011 07:45:53.592802207 ProcessGroupNCCL.cpp:760] [Rank 11] To avoid data inconsistency, we are taking the entire process down. +[rank15]:[E1011 07:45:53.594603652 ProcessGroupNCCL.cpp:746] [Rank 15] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank15]:[E1011 07:45:53.594633445 ProcessGroupNCCL.cpp:760] [Rank 15] To avoid data inconsistency, we are taking the entire process down. +[rank11]:[E1011 07:45:53.595776254 ProcessGroupNCCL.cpp:2068] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] Process group watchdog thread terminated with exception: [Rank 11] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200075 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f0df16d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f0d94ca0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f0d94ca3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f0d94ca4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f0d78601db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f0df28ffaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f0df298cc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank15]:[E1011 07:45:53.595857274 ProcessGroupNCCL.cpp:2068] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] Process group watchdog thread terminated with exception: [Rank 15] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200076 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f6af6f7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f6a9a4a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f6a9a4a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f6a9a4a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f6a7de01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f6af8156aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f6af81e3c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] Process group watchdog thread terminated with exception: [Rank 11] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200075 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f0df16d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f0d94ca0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f0d94ca3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f0d94ca4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f0d78601db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f0df28ffaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f0df298cc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f0df16d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7f0d94c7c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7f0d947a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7f0d78601db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7f0df28ffaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7f0df298cc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + + what(): [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] Process group watchdog thread terminated with exception: [Rank 15] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200076 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f6af6f7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f6a9a4a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f6a9a4a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f6a9a4a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f6a7de01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f6af8156aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f6af81e3c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f6af6f7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7f6a9a47c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7f6a99fa68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7f6a7de01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7f6af8156aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7f6af81e3c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank9]:[E1011 07:45:54.605992637 ProcessGroupNCCL.cpp:746] [Rank 9] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank9]:[E1011 07:45:54.606007967 ProcessGroupNCCL.cpp:760] [Rank 9] To avoid data inconsistency, we are taking the entire process down. +[rank9]:[E1011 07:45:54.606885496 ProcessGroupNCCL.cpp:2068] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] Process group watchdog thread terminated with exception: [Rank 9] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200077 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbd8c37eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7fbd2f8a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7fbd2f8a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fbd2f8a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7fbd13201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7fbd8d581aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7fbd8d60ec3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] Process group watchdog thread terminated with exception: [Rank 9] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200077 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbd8c37eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7fbd2f8a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7fbd2f8a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fbd2f8a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7fbd13201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7fbd8d581aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7fbd8d60ec3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbd8c37eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7fbd2f87c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7fbd2f3a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7fbd13201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7fbd8d581aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7fbd8d60ec3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank13]:[E1011 07:45:54.610026526 ProcessGroupNCCL.cpp:746] [Rank 5] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank13]:[E1011 07:45:54.610043013 ProcessGroupNCCL.cpp:760] [Rank 5] To avoid data inconsistency, we are taking the entire process down. +[rank13]:[E1011 07:45:54.610968779 ProcessGroupNCCL.cpp:2068] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] Process group watchdog thread terminated with exception: [Rank 5] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200035 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f9fc40d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f9f676a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f9f676a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f9f676a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f9f4b001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f9fc5344aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f9fc53d1c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] Process group watchdog thread terminated with exception: [Rank 5] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200035 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f9fc40d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f9f676a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f9f676a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f9f676a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f9f4b001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f9fc5344aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f9fc53d1c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f9fc40d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7f9f6767c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7f9f671a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7f9f4b001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7f9fc5344aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7f9fc53d1c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank14]:[E1011 07:45:54.642326310 ProcessGroupNCCL.cpp:746] [Rank 14] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank14]:[E1011 07:45:54.642385861 ProcessGroupNCCL.cpp:760] [Rank 14] To avoid data inconsistency, we are taking the entire process down. +[rank14]:[E1011 07:45:54.643253362 ProcessGroupNCCL.cpp:2068] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] Process group watchdog thread terminated with exception: [Rank 14] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200062 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f7c3317eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f7bd66a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f7bd66a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f7bd66a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f7bba001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f7c3440baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f7c34498c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] Process group watchdog thread terminated with exception: [Rank 14] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200062 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f7c3317eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f7bd66a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f7bd66a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f7bd66a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f7bba001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f7c3440baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f7c34498c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f7c3317eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7f7bd667c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7f7bd61a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7f7bba001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7f7c3440baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7f7c34498c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank8]:[E1011 07:45:54.686805223 ProcessGroupNCCL.cpp:746] [Rank 0] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank8]:[E1011 07:45:54.686821876 ProcessGroupNCCL.cpp:760] [Rank 0] To avoid data inconsistency, we are taking the entire process down. +[rank8]:[E1011 07:45:54.687700494 ProcessGroupNCCL.cpp:2068] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] Process group watchdog thread terminated with exception: [Rank 0] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200021 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fd9c9cd9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7fd96d2a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7fd96d2a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fd96d2a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7fd950c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7fd9caf42aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7fd9cafcfc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] Process group watchdog thread terminated with exception: [Rank 0] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200021 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fd9c9cd9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7fd96d2a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7fd96d2a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fd96d2a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7fd950c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7fd9caf42aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7fd9cafcfc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fd9c9cd9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7fd96d27c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7fd96cda68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7fd950c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7fd9caf42aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7fd9cafcfc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank10]:[E1011 07:45:54.690505249 ProcessGroupNCCL.cpp:746] [Rank 10] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank10]:[E1011 07:45:54.690527795 ProcessGroupNCCL.cpp:760] [Rank 10] To avoid data inconsistency, we are taking the entire process down. +[rank12]:[E1011 07:45:54.691423893 ProcessGroupNCCL.cpp:746] [Rank 4] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank10]:[E1011 07:45:54.691435729 ProcessGroupNCCL.cpp:2068] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] Process group watchdog thread terminated with exception: [Rank 10] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200035 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f62925deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f62934a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f62934a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f62934a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f6276c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f62f105baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f62f10e8c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank12]:[E1011 07:45:54.691439617 ProcessGroupNCCL.cpp:760] [Rank 4] To avoid data inconsistency, we are taking the entire process down. +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] Process group watchdog thread terminated with exception: [Rank 10] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200035 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f62925deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f62934a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f62934a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f62934a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f6276c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f62f105baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f62f10e8c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f62925deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7f629347c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7f6292fa68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7f6276c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7f62f105baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7f62f10e8c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank12]:[E1011 07:45:54.692382694 ProcessGroupNCCL.cpp:2068] [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] Process group watchdog thread terminated with exception: [Rank 4] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200026 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f4e705deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f4e714a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f4e714a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f4e714a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f4e54c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f4ecf057aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f4ecf0e4c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 14 PG GUID 453(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] Process group watchdog thread terminated with exception: [Rank 4] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52272, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200026 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f4e705deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f4e714a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f4e714a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f4e714a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f4e54c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f4ecf057aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f4ecf0e4c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f4e705deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7f4e7147c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7f4e70fa68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7f4e54c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7f4ecf057aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7f4ecf0e4c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +W1011 07:45:59.280000 44076 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49471 closing signal SIGTERM +W1011 07:45:59.291000 44076 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49474 closing signal SIGTERM +W1011 07:45:59.293000 44076 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49476 closing signal SIGTERM +W1011 07:45:59.295000 44076 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49477 closing signal SIGTERM +E1011 07:46:02.519000 44076 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -6) local_rank: 1 (pid: 49472) of binary: /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +Traceback (most recent call last): + File "/mnt/ssd/yulan/cache/installs/cpython-3.10.19-linux-x86_64-gnu/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/mnt/ssd/yulan/cache/installs/cpython-3.10.19-linux-x86_64-gnu/lib/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 905, in + main() + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 143, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 277, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-10-11_07:45:59 + host : cs-bca8a-46a8d-server + rank : 10 (local_rank: 2) + exitcode : -6 (pid: 49473) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 49473 +[2]: + time : 2026-10-11_07:45:59 + host : cs-bca8a-46a8d-server + rank : 12 (local_rank: 4) + exitcode : -6 (pid: 49475) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 49475 +[3]: + time : 2026-10-11_07:45:59 + host : cs-bca8a-46a8d-server + rank : 15 (local_rank: 7) + exitcode : -6 (pid: 49478) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 49478 +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-10-11_07:45:59 + host : cs-bca8a-46a8d-server + rank : 9 (local_rank: 1) + exitcode : -6 (pid: 49472) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 49472 +============================================================ diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node2.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node2.log new file mode 100644 index 0000000000000000000000000000000000000000..eb17a1ae26ce3c017b531ad7ca1d53f06ed7da7a --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node2.log @@ -0,0 +1,782 @@ +W1011 02:40:44.504000 104749 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:44.504000 104749 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:45.415000 104742 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:45.415000 104742 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:45.572000 104745 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:45.572000 104745 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:45.811000 104747 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:45.811000 104747 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +W1011 02:40:45.852000 104743 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:45.852000 104743 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank [Gloo] Rank 19 is connected to 2163 is connected to peer ranks. 63Expected number of connected peer ranks is : peer ranks. 63Expected number of connected peer ranks is : +63 +[Gloo] Rank 22 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 18 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank21]:[W1011 02:41:49.049199668 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank21]:[W1011 02:41:49.049210859 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank19]:[W1011 02:41:49.049207638 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank19]:[W1011 02:41:49.049219105 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 17 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank18]:[W1011 02:41:49.049309759 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank18]:[W1011 02:41:49.049319703 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank22]:[W1011 02:41:49.049320247 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank22]:[W1011 02:41:49.049332099 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 23 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank17]:[W1011 02:41:49.049419377 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank17]:[W1011 02:41:49.049433476 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank23]:[W1011 02:41:49.049493974 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank23]:[W1011 02:41:49.049504234 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 16 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank16]:[W1011 02:41:49.051569563 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank16]:[W1011 02:41:49.051580108 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 20 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank20]:[W1011 02:41:50.058660775 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank20]:[W1011 02:41:50.058674963 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank17]:[W1011 02:41:51.166854211 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 17] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank23]:[W1011 02:41:51.169040866 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 23] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank16]:[W1011 02:41:51.169739534 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 16] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank19]:[W1011 02:41:52.176293672 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 19] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank18]:[W1011 02:41:52.177294776 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 18] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank21]:[W1011 02:41:52.180107260 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 21] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank22]:[W1011 02:41:52.180637773 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 22] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank20]:[W1011 02:41:52.181302228 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 20] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank17]:[W1011 02:41:52.951903222 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank17]:[W1011 02:41:52.951932785 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank16]:[W1011 02:41:52.951991411 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank16]:[W1011 02:41:52.952016846 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank18]:[W1011 02:41:52.952050913 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank19]:[W1011 02:41:52.952060180 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank18]:[W1011 02:41:52.952072667 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank19]:[W1011 02:41:52.952075425 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank23]:[W1011 02:41:52.952077112 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank23]:[W1011 02:41:52.952092596 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank20]:[W1011 02:41:52.952098551 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank20]:[W1011 02:41:52.952113103 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank21]:[W1011 02:41:52.952153138 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank21]:[W1011 02:41:52.952172370 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank22]:[W1011 02:41:52.952232626 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank22]:[W1011 02:41:52.952247390 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0 + + +>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset>>> start_samples: 0 + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 + +>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +>>> start_samples: 0Initializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +>>> start_samples: 0 +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +W1011 04:40:53.694000 250979 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 04:40:53.694000 250979 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank17]:[E1011 07:33:47.556236484 ProcessGroupNCCL.cpp:685] [Rank 17] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200000 milliseconds before timing out. +[rank20]:[E1011 07:33:47.556240128 ProcessGroupNCCL.cpp:685] [Rank 20] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200001 milliseconds before timing out. +[rank19]:[E1011 07:33:47.567840369 ProcessGroupNCCL.cpp:685] [Rank 19] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200020 milliseconds before timing out. +[rank17]:[E1011 07:33:47.568231339 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 17] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank19]:[E1011 07:33:47.568236391 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 19] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank20]:[E1011 07:33:47.568236065 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 20] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank17]:[E1011 07:33:47.569088424 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank20]:[E1011 07:33:47.569093275 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank19]:[E1011 07:33:47.569094924 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank19]:[E1011 07:33:47.572061574 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 19] First PG on this rank to signal dumping. +[rank17]:[E1011 07:33:47.572064730 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 17] First PG on this rank to signal dumping. +[rank20]:[E1011 07:33:47.572070128 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 20] First PG on this rank to signal dumping. +[rank21]:[E1011 07:33:47.575545415 ProcessGroupNCCL.cpp:685] [Rank 21] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200028 milliseconds before timing out. +[rank21]:[E1011 07:33:47.575657777 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 21] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank21]:[E1011 07:33:47.575669428 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank21]:[E1011 07:33:47.575744824 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 21] First PG on this rank to signal dumping. +[rank16]:[E1011 07:33:47.583623138 ProcessGroupNCCL.cpp:685] [Rank 16] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200036 milliseconds before timing out. +[rank16]:[E1011 07:33:47.583726668 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 16] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank16]:[E1011 07:33:47.583742370 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank16]:[E1011 07:33:47.583816432 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 16] First PG on this rank to signal dumping. +[rank23]:[E1011 07:33:47.596514578 ProcessGroupNCCL.cpp:685] [Rank 23] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200049 milliseconds before timing out. +[rank23]:[E1011 07:33:47.596615155 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 23] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank23]:[E1011 07:33:47.596631375 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank23]:[E1011 07:33:47.596699770 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 23] First PG on this rank to signal dumping. +[rank22]:[E1011 07:33:47.621543503 ProcessGroupNCCL.cpp:685] [Rank 22] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200074 milliseconds before timing out. +[rank22]:[E1011 07:33:47.621659464 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 22] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank22]:[E1011 07:33:47.621677562 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank22]:[E1011 07:33:47.621798689 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 22] First PG on this rank to signal dumping. +[rank18]:[E1011 07:33:47.645093726 ProcessGroupNCCL.cpp:685] [Rank 18] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200097 milliseconds before timing out. +[rank18]:[E1011 07:33:47.645199519 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 18] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank18]:[E1011 07:33:47.645212123 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank18]:[E1011 07:33:47.645283981 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 18] First PG on this rank to signal dumping. +[rank16]:[E1011 07:33:47.883368184 ProcessGroupNCCL.cpp:685] [Rank 16] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200025 milliseconds before timing out. +[rank16]:[E1011 07:33:47.883403451 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 454(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank16]:[E1011 07:33:47.883412119 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank17]:[E1011 07:33:47.890214004 ProcessGroupNCCL.cpp:685] [Rank 17] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200025 milliseconds before timing out. +[rank17]:[E1011 07:33:47.890239770 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 454(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank17]:[E1011 07:33:47.890246639 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank21]:[E1011 07:33:47.890905799 ProcessGroupNCCL.cpp:685] [Rank 21] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200022 milliseconds before timing out. +[rank21]:[E1011 07:33:47.890940119 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 454(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank21]:[E1011 07:33:47.890947474 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank19]:[E1011 07:33:47.933107723 ProcessGroupNCCL.cpp:685] [Rank 19] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200062 milliseconds before timing out. +[rank19]:[E1011 07:33:47.933145674 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 454(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank19]:[E1011 07:33:47.933152861 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank20]:[E1011 07:33:47.943332104 ProcessGroupNCCL.cpp:685] [Rank 20] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200075 milliseconds before timing out. +[rank20]:[E1011 07:33:47.943373654 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 454(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank20]:[E1011 07:33:47.943383244 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank22]:[E1011 07:33:47.953510902 ProcessGroupNCCL.cpp:685] [Rank 22] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200085 milliseconds before timing out. +[rank22]:[E1011 07:33:47.953547977 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 454(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank22]:[E1011 07:33:47.953557095 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank18]:[E1011 07:33:47.959602011 ProcessGroupNCCL.cpp:685] [Rank 18] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200093 milliseconds before timing out. +[rank18]:[E1011 07:33:47.959644715 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 454(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank18]:[E1011 07:33:47.959652050 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank23]:[E1011 07:33:47.967555511 ProcessGroupNCCL.cpp:685] [Rank 23] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200098 milliseconds before timing out. +[rank23]:[E1011 07:33:47.967599303 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 454(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank23]:[E1011 07:33:47.967610712 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank19]:[W1011 07:33:47.933250121 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=97, addr=[cs-ce73c-5a5bd-server]:33428, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f71776d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f715b7bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f715b7bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f715b7bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f715b7b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f711ac95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f711ac9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f711aca2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f711aca4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f70fe601db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f717893baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f71789c8c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank21]:[W1011 07:33:47.891023490 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=97, addr=[cs-ce73c-5a5bd-server]:33454, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ff05957eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7ff03d5bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7ff03d5bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7ff03d5bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7ff03d5b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7feffca95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7feffca9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7feffcaa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7feffcaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fefe0401db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7ff05a82eaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7ff05a8bbc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank22]:[W1011 07:33:47.953626422 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=97, addr=[cs-ce73c-5a5bd-server]:33438, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fa30477eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fa2e87bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fa2e87bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fa2e87bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fa2e87b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fa2a7c95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fa2a7c9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fa2a7ca2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fa2a7ca4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fa28b601db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fa305a19aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fa305aa6c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank17]:[W1011 07:33:47.890325047 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=97, addr=[cs-ce73c-5a5bd-server]:33446, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f39d5d7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f39b9dbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f39b9dbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f39b9dbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f39b9db71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f3979295880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f397929599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f39792a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f39792a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f395cc01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f39d6ffbaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f39d7088c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank20]:[W1011 07:33:47.943456863 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=97, addr=[cs-ce73c-5a5bd-server]:33422, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbe87d7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fbe6bdbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fbe6bdbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fbe6bdbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fbe6bdb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fbe2b295880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fbe2b29599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fbe2b2a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fbe2b2a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fbe0ec01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fbe88f8faa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fbe8901cc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank23]:[W1011 07:33:47.967697403 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=97, addr=[cs-ce73c-5a5bd-server]:33400, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fee60d7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fee44dbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fee44dbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fee44dbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fee44db71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fee04295880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fee0429599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fee042a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fee042a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fede7c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fee61faeaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fee6203bc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank16]:[W1011 07:33:47.889725394 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=97, addr=[cs-ce73c-5a5bd-server]:33406, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f62d60ceeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f62ba1bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f62ba1bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f62ba1bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f62ba1b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f6279695880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f627969599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f62796a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f62796a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f625d001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f62d729caa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f62d7329c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank18]:[W1011 07:33:47.959725294 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=97, addr=[cs-ce73c-5a5bd-server]:33450, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f0dcad7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f0daedbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f0daedbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f0daedbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f0daedb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f0d6e295880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f0d6e29599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f0d6e2a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f0d6e2a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f0d51c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f0dcbff8aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f0dcc085c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank21]:[E1011 07:33:47.990878056 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 454(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ff05957eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7ff03d5bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7ff03d5bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7ff03d5bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7ff03d5b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7feffca95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7feffca9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7feffcaa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7feffcaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fefe0401db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7ff05a82eaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7ff05a8bbc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank22]:[E1011 07:33:47.990882618 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 454(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fa30477eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fa2e87bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fa2e87bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fa2e87bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fa2e87b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fa2a7c95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fa2a7c9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fa2a7ca2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fa2a7ca4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fa28b601db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fa305a19aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fa305aa6c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank19]:[E1011 07:33:47.990884725 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 454(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f71776d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f715b7bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f715b7bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f715b7bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f715b7b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f711ac95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f711ac9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f711aca2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f711aca4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f70fe601db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f717893baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f71789c8c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank17]:[E1011 07:33:47.990885686 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 454(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f39d5d7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f39b9dbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f39b9dbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f39b9dbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f39b9db71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f3979295880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f397929599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f39792a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f39792a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f395cc01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f39d6ffbaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f39d7088c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank20]:[E1011 07:33:47.990891732 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 454(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbe87d7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fbe6bdbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fbe6bdbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fbe6bdbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fbe6bdb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fbe2b295880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fbe2b29599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fbe2b2a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fbe2b2a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fbe0ec01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fbe88f8faa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fbe8901cc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank18]:[E1011 07:33:47.990891532 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 454(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f0dcad7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f0daedbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f0daedbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f0daedbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f0daedb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f0d6e295880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f0d6e29599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f0d6e2a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f0d6e2a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f0d51c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f0dcbff8aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f0dcc085c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank16]:[E1011 07:33:47.990896968 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 454(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f62d60ceeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f62ba1bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f62ba1bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f62ba1bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f62ba1b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f6279695880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f627969599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f62796a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f62796a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f625d001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f62d729caa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f62d7329c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank23]:[E1011 07:33:47.990896934 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 454(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fee60d7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fee44dbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fee44dbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fee44dbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fee44db71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fee04295880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fee0429599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fee042a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fee042a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fede7c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fee61faeaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fee6203bc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank23]:[E1011 07:34:47.702576945 ProcessGroupNCCL.cpp:746] [Rank 23] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank23]:[E1011 07:34:47.702629589 ProcessGroupNCCL.cpp:760] [Rank 23] To avoid data inconsistency, we are taking the entire process down. +[rank23]:[E1011 07:34:47.706987872 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 23] Process group watchdog thread terminated with exception: [Rank 23] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200049 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fee60d7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7fee042a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7fee042a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fee042a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7fede7c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7fee61faeaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7fee6203bc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank20]:[E1011 07:34:47.707229328 ProcessGroupNCCL.cpp:746] [Rank 20] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank20]:[E1011 07:34:47.707253836 ProcessGroupNCCL.cpp:760] [Rank 20] To avoid data inconsistency, we are taking the entire process down. +[rank20]:[E1011 07:34:47.708193992 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 20] Process group watchdog thread terminated with exception: [Rank 20] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200001 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbe87d7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7fbe2b2a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7fbe2b2a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fbe2b2a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7fbe0ec01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7fbe88f8faa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7fbe8901cc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 23] Process group watchdog thread terminated with exception: [Rank 23] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200049 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fee60d7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7fee042a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7fee042a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fee042a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7fede7c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7fee61faeaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7fee6203bc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fee60d7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7fee0427c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7fee03da68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7fede7c01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7fee61faeaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7fee6203bc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 20] Process group watchdog thread terminated with exception: [Rank 20] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200001 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbe87d7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7fbe2b2a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7fbe2b2a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fbe2b2a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7fbe0ec01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7fbe88f8faa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7fbe8901cc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbe87d7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7fbe2b27c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7fbe2ada68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7fbe0ec01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7fbe88f8faa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7fbe8901cc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +W1011 07:34:50.740000 101059 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 104742 closing signal SIGTERM +W1011 07:34:50.762000 101059 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 104743 closing signal SIGTERM +W1011 07:34:50.777000 101059 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 104744 closing signal SIGTERM +W1011 07:34:50.782000 101059 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 104745 closing signal SIGTERM +W1011 07:34:50.802000 101059 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 104747 closing signal SIGTERM +W1011 07:34:50.807000 101059 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 104748 closing signal SIGTERM +W1011 07:35:20.830000 101059 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 104742 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:23.398000 101059 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 104743 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:25.978000 101059 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 104744 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:28.554000 101059 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 104745 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:31.187000 101059 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 104747 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:33.985000 101059 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 104748 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +E1011 07:35:36.831000 101059 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -6) local_rank: 4 (pid: 104746) of binary: /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +Traceback (most recent call last): + File "/mnt/ssd/yulan/cache/installs/cpython-3.10.19-linux-x86_64-gnu/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/mnt/ssd/yulan/cache/installs/cpython-3.10.19-linux-x86_64-gnu/lib/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 905, in + main() + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 143, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 277, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-10-11_07:34:50 + host : cs-ce73c-5a5bd-server + rank : 23 (local_rank: 7) + exitcode : -6 (pid: 104749) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 104749 +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-10-11_07:34:50 + host : cs-ce73c-5a5bd-server + rank : 20 (local_rank: 4) + exitcode : -6 (pid: 104746) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 104746 +============================================================ diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node3.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node3.log new file mode 100644 index 0000000000000000000000000000000000000000..ce253bde642df688d3323cd89d05212b6366adb7 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node3.log @@ -0,0 +1,917 @@ +W1011 02:41:10.043000 27985 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:41:10.043000 27985 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank 24 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 25 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 26 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank25]:[W1011 02:41:50.917495705 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank24]:[W1011 02:41:50.917495705 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank26]:[W1011 02:41:50.917506461 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank25]:[W1011 02:41:50.917516449 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank24]:[W1011 02:41:50.917521268 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank26]:[W1011 02:41:50.917523713 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 27 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank27]:[W1011 02:41:50.918010664 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank27]:[W1011 02:41:50.918030081 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 28 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 29 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank29]:[W1011 02:41:50.918545109 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank29]:[W1011 02:41:50.918561523 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank28]:[W1011 02:41:50.918574933 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank28]:[W1011 02:41:50.918598191 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 30 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank30]:[W1011 02:41:50.919333507 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank30]:[W1011 02:41:50.919350199 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 31 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank31]:[W1011 02:41:50.919815010 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank31]:[W1011 02:41:50.919836452 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 1 is connected to 3 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 1[Gloo] Rank 3 is connected to 1 +3 is connected to [Gloo] Rank peer ranks. 31Expected number of connected peer ranks is : peer ranks. is connected to 3Expected number of connected peer ranks is : 3 +3[Gloo] Rank peer ranks. +1Expected number of connected peer ranks is : [Gloo] Rank is connected to 313 + is connected to peer ranks. 3Expected number of connected peer ranks is : peer ranks. 3Expected number of connected peer ranks is : +3 +[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank24]:[W1011 02:41:52.984818809 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 24] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank25]:[W1011 02:41:52.986235661 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 25] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank31]:[W1011 02:41:52.990282204 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 31] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank26]:[W1011 02:41:52.991477861 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 26] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank27]:[W1011 02:41:52.992576644 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 27] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank28]:[W1011 02:41:52.997105040 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 28] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank30]:[W1011 02:41:52.997464177 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 30] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank29]:[W1011 02:41:52.997843359 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 29] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank30]:[W1011 02:41:52.800186016 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank30]:[W1011 02:41:52.800230716 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank24]:[W1011 02:41:52.800627709 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank24]:[W1011 02:41:52.800656973 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank29]:[W1011 02:41:52.801128280 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank29]:[W1011 02:41:52.801150769 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank26]:[W1011 02:41:52.801167462 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank28]:[W1011 02:41:52.801273485 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank26]:[W1011 02:41:52.801293879 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank28]:[W1011 02:41:52.801301632 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank31]:[W1011 02:41:52.801384467 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank31]:[W1011 02:41:52.801421694 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank25]:[W1011 02:41:52.801809676 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank25]:[W1011 02:41:52.801833004 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank27]:[W1011 02:41:52.802075223 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank27]:[W1011 02:41:52.802097084 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +>>> start_samples: 0 +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0Initializing GlobIndexedDataset + + + +>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +>>> start_samples: 0 +Initializing GlobIndexedDataset +>>> start_samples: 0 +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + + +pad_document_marker: 12051293 +pad_document_marker: 12051293 +pad_document_marker: 12051293 +pad_document_marker: 12051293pad_document_marker: 12051293 + +pad_document_marker: 12051293 +pad_document_marker: 12051293 +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True + +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDataset + + +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDataset + + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True + + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + + + + + +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +W1011 04:41:12.663000 22152 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 04:41:12.663000 22152 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank25]:[E1011 07:33:47.280831080 ProcessGroupNCCL.cpp:685] [Rank 25] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200007 milliseconds before timing out. +[rank29]:[E1011 07:33:47.282665503 ProcessGroupNCCL.cpp:685] [Rank 29] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200030 milliseconds before timing out. +[rank27]:[E1011 07:33:47.283923443 ProcessGroupNCCL.cpp:685] [Rank 27] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200031 milliseconds before timing out. +[rank31]:[E1011 07:33:47.284616633 ProcessGroupNCCL.cpp:685] [Rank 31] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200031 milliseconds before timing out. +[rank28]:[E1011 07:33:47.291334120 ProcessGroupNCCL.cpp:685] [Rank 28] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200038 milliseconds before timing out. +[rank24]:[E1011 07:33:47.298773643 ProcessGroupNCCL.cpp:685] [Rank 24] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200045 milliseconds before timing out. +[rank27]:[E1011 07:33:47.302591324 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 27] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank29]:[E1011 07:33:47.302608855 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 29] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank25]:[E1011 07:33:47.302605432 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 25] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank31]:[E1011 07:33:47.302621287 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 31] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank28]:[E1011 07:33:47.302615140 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 28] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank24]:[E1011 07:33:47.302633579 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 24] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank29]:[E1011 07:33:47.305296967 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank25]:[E1011 07:33:47.305314637 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank27]:[E1011 07:33:47.305320783 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank28]:[E1011 07:33:47.305331399 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank31]:[E1011 07:33:47.305343692 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank24]:[E1011 07:33:47.305370511 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank29]:[E1011 07:33:47.312154629 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 29] First PG on this rank to signal dumping. +[rank31]:[E1011 07:33:47.312184242 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 31] First PG on this rank to signal dumping. +[rank25]:[E1011 07:33:47.312201284 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 25] First PG on this rank to signal dumping. +[rank28]:[E1011 07:33:47.312214274 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 28] First PG on this rank to signal dumping. +[rank27]:[E1011 07:33:47.312225100 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 27] First PG on this rank to signal dumping. +[rank24]:[E1011 07:33:47.312248148 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 24] First PG on this rank to signal dumping. +[rank26]:[E1011 07:33:47.330848701 ProcessGroupNCCL.cpp:685] [Rank 26] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200077 milliseconds before timing out. +[rank26]:[E1011 07:33:47.331097411 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 26] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank26]:[E1011 07:33:47.331130168 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank26]:[E1011 07:33:47.331263567 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 26] First PG on this rank to signal dumping. +[rank30]:[E1011 07:33:47.343281919 ProcessGroupNCCL.cpp:685] [Rank 30] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200089 milliseconds before timing out. +[rank30]:[E1011 07:33:47.343566179 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 30] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank30]:[E1011 07:33:47.343614860 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank30]:[E1011 07:33:47.343786184 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 30] First PG on this rank to signal dumping. +[rank29]:[E1011 07:33:47.581977743 ProcessGroupNCCL.cpp:685] [Rank 29] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200000 milliseconds before timing out. +[rank29]:[E1011 07:33:47.582158287 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 455(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank29]:[E1011 07:33:47.582177354 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank31]:[E1011 07:33:47.584532176 ProcessGroupNCCL.cpp:685] [Rank 31] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200003 milliseconds before timing out. +[rank31]:[E1011 07:33:47.584574082 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 455(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank31]:[E1011 07:33:47.584593009 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank30]:[E1011 07:33:47.585039026 ProcessGroupNCCL.cpp:685] [Rank 30] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200003 milliseconds before timing out. +[rank30]:[E1011 07:33:47.585144349 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 455(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank30]:[E1011 07:33:47.585159644 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank26]:[E1011 07:33:47.587905237 ProcessGroupNCCL.cpp:685] [Rank 26] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200006 milliseconds before timing out. +[rank26]:[E1011 07:33:47.588066504 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 455(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank26]:[E1011 07:33:47.588089902 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank25]:[E1011 07:33:47.592654762 ProcessGroupNCCL.cpp:685] [Rank 25] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200010 milliseconds before timing out. +[rank25]:[E1011 07:33:47.592718528 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 455(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank25]:[E1011 07:33:47.592733335 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank24]:[E1011 07:33:47.598789829 ProcessGroupNCCL.cpp:685] [Rank 24] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200018 milliseconds before timing out. +[rank24]:[E1011 07:33:47.598927420 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 455(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank24]:[E1011 07:33:47.598952703 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank27]:[E1011 07:33:47.627595682 ProcessGroupNCCL.cpp:685] [Rank 27] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200046 milliseconds before timing out. +[rank27]:[E1011 07:33:47.627699119 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 455(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank27]:[E1011 07:33:47.627714066 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank28]:[E1011 07:33:47.647771890 ProcessGroupNCCL.cpp:685] [Rank 28] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200064 milliseconds before timing out. +[rank28]:[E1011 07:33:47.647967939 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 455(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank28]:[E1011 07:33:47.647984980 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank28]:[W1011 07:33:47.648240256 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=97, addr=[cs-54361-84575-server]:56538, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fa1efbdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fa2315bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fa2315bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fa2315bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fa2315b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fa1f0a95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fa1f0a9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fa1f0aa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fa1f0aa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fa1d4201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fa24e653aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fa24e6e0c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank29]:[W1011 07:33:47.598864002 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=97, addr=[cs-54361-84575-server]:56572, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbd4037eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fbd243bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fbd243bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fbd243bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fbd243b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fbce3895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fbce389599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fbce38a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fbce38a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fbcc7201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fbd41648aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fbd416d5c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank31]:[W1011 07:33:47.598931261 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=97, addr=[cs-54361-84575-server]:56550, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f51c34d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f51a75bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f51a75bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f51a75bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f51a75b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f5166a95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f5166a9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f5166aa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f5166aa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f514a401db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f51c46e9aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f51c4776c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank26]:[W1011 07:33:47.598890752 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=97, addr=[cs-54361-84575-server]:56510, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f2e303deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f2e71dbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f2e71dbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f2e71dbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f2e71db71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f2e31295880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f2e3129599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f2e312a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f2e312a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f2e14a01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f2e8ee7baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f2e8ef08c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank27]:[W1011 07:33:47.627908718 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=97, addr=[cs-54361-84575-server]:56498, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f61d19deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f62133bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f62133bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f62133bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f62133b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f61d2895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f61d289599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f61d28a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f61d28a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f61b6001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f623047aaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f6230507c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank24]:[W1011 07:33:47.599376369 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=97, addr=[cs-54361-84575-server]:56564, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f81784d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f815c5bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f815c5bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f815c5bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f815c5b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f811ba95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f811ba9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f811baa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f811baa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f80ff401db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f81796c4aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f8179751c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank30]:[W1011 07:33:47.598875107 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=97, addr=[cs-54361-84575-server]:56516, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f85aa97eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f858e9bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f858e9bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f858e9bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f858e9b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f854de95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f854de9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f854dea2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f854dea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f8531801db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f85abc1baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f85abca8c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank25]:[W1011 07:33:47.598912753 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=97, addr=[cs-54361-84575-server]:56522, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fd388ed9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fd36cfbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fd36cfbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fd36cfbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fd36cfb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fd32c495880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fd32c49599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fd32c4a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fd32c4a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fd30fe01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fd38a0d1aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fd38a15ec3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank25]:[E1011 07:33:47.773686219 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 455(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fd388ed9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fd36cfbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fd36cfbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fd36cfbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fd36cfb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fd32c495880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fd32c49599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fd32c4a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fd32c4a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fd30fe01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fd38a0d1aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fd38a15ec3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank29]:[E1011 07:33:47.773711083 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 455(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbd4037eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fbd243bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fbd243bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fbd243bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fbd243b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fbce3895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fbce389599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fbce38a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fbce38a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fbcc7201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fbd41648aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fbd416d5c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank24]:[E1011 07:33:47.773724493 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 455(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f81784d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f815c5bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f815c5bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f815c5bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f815c5b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f811ba95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f811ba9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f811baa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f811baa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f80ff401db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f81796c4aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f8179751c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank31]:[E1011 07:33:47.773728823 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 455(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f51c34d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f51a75bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f51a75bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f51a75bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f51a75b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f5166a95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f5166a9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f5166aa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f5166aa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f514a401db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f51c46e9aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f51c4776c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank30]:[E1011 07:33:47.773739648 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 455(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f85aa97eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f858e9bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f858e9bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f858e9bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f858e9b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f854de95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f854de9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f854dea2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f854dea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f8531801db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f85abc1baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f85abca8c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank26]:[E1011 07:33:47.773742023 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 455(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f2e303deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f2e71dbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f2e71dbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f2e71dbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f2e71db71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f2e31295880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f2e3129599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f2e312a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f2e312a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f2e14a01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f2e8ee7baa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f2e8ef08c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank28]:[E1011 07:33:47.773754455 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 455(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fa1efbdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fa2315bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fa2315bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fa2315bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fa2315b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fa1f0a95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fa1f0a9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fa1f0aa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fa1f0aa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fa1d4201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fa24e653aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fa24e6e0c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank27]:[E1011 07:33:47.773770170 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 455(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f61d19deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f62133bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f62133bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f62133bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f62133b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f61d2895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f61d289599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f61d28a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f61d28a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f61b6001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f623047aaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f6230507c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank28]:[E1011 07:34:47.605052200 ProcessGroupNCCL.cpp:746] [Rank 28] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank28]:[E1011 07:34:47.605164717 ProcessGroupNCCL.cpp:760] [Rank 28] To avoid data inconsistency, we are taking the entire process down. +[rank28]:[E1011 07:34:47.609404599 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 28] Process group watchdog thread terminated with exception: [Rank 28] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200038 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fa1efbdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7fa1f0aa0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7fa1f0aa3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fa1f0aa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7fa1d4201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7fa24e653aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7fa24e6e0c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 28] Process group watchdog thread terminated with exception: [Rank 28] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200038 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fa1efbdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7fa1f0aa0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7fa1f0aa3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fa1f0aa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7fa1d4201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7fa24e653aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7fa24e6e0c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fa1efbdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7fa1f0a7c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7fa1f05a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7fa1d4201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7fa24e653aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7fa24e6e0c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank29]:[E1011 07:34:47.660845841 ProcessGroupNCCL.cpp:746] [Rank 29] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank29]:[E1011 07:34:47.660874965 ProcessGroupNCCL.cpp:760] [Rank 29] To avoid data inconsistency, we are taking the entire process down. +[rank29]:[E1011 07:34:47.661734103 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 29] Process group watchdog thread terminated with exception: [Rank 29] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200030 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbd4037eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7fbce38a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7fbce38a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fbce38a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7fbcc7201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7fbd41648aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7fbd416d5c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 29] Process group watchdog thread terminated with exception: [Rank 29] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200030 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbd4037eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7fbce38a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7fbce38a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fbce38a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7fbcc7201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7fbd41648aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7fbd416d5c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbd4037eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7fbce387c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7fbce33a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7fbcc7201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7fbd41648aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7fbd416d5c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank31]:[E1011 07:34:47.689579973 ProcessGroupNCCL.cpp:746] [Rank 31] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank31]:[E1011 07:34:47.689621878 ProcessGroupNCCL.cpp:760] [Rank 31] To avoid data inconsistency, we are taking the entire process down. +[rank31]:[E1011 07:34:47.690780502 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 31] Process group watchdog thread terminated with exception: [Rank 31] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200031 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f51c34d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f5166aa0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f5166aa3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f5166aa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f514a401db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f51c46e9aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f51c4776c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 31] Process group watchdog thread terminated with exception: [Rank 31] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200031 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f51c34d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f5166aa0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f5166aa3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f5166aa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f514a401db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f51c46e9aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f51c4776c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f51c34d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7f5166a7c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7f51665a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7f514a401db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7f51c46e9aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7f51c4776c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank24]:[E1011 07:34:47.693650765 ProcessGroupNCCL.cpp:746] [Rank 24] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank24]:[E1011 07:34:47.693693579 ProcessGroupNCCL.cpp:760] [Rank 24] To avoid data inconsistency, we are taking the entire process down. +[rank24]:[E1011 07:34:47.694716428 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 24] Process group watchdog thread terminated with exception: [Rank 24] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200045 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f81784d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f811baa0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f811baa3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f811baa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f80ff401db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f81796c4aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f8179751c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 24] Process group watchdog thread terminated with exception: [Rank 24] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200045 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f81784d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f811baa0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f811baa3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f811baa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f80ff401db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f81796c4aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f8179751c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f81784d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7f811ba7c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7f811b5a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7f80ff401db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7f81796c4aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7f8179751c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank25]:[E1011 07:34:47.758438005 ProcessGroupNCCL.cpp:746] [Rank 25] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank25]:[E1011 07:34:47.758479911 ProcessGroupNCCL.cpp:760] [Rank 25] To avoid data inconsistency, we are taking the entire process down. +[rank25]:[E1011 07:34:47.759400230 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 25] Process group watchdog thread terminated with exception: [Rank 25] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200007 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fd388ed9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7fd32c4a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7fd32c4a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fd32c4a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7fd30fe01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7fd38a0d1aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7fd38a15ec3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 25] Process group watchdog thread terminated with exception: [Rank 25] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200007 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fd388ed9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7fd32c4a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7fd32c4a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fd32c4a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7fd30fe01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7fd38a0d1aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7fd38a15ec3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fd388ed9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7fd32c47c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7fd32bfa68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7fd30fe01db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7fd38a0d1aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7fd38a15ec3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank27]:[E1011 07:34:47.801427000 ProcessGroupNCCL.cpp:746] [Rank 27] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank27]:[E1011 07:34:47.801481897 ProcessGroupNCCL.cpp:760] [Rank 27] To avoid data inconsistency, we are taking the entire process down. +[rank27]:[E1011 07:34:47.802503558 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 27] Process group watchdog thread terminated with exception: [Rank 27] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200031 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f61d19deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f61d28a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f61d28a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f61d28a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f61b6001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f623047aaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f6230507c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 27] Process group watchdog thread terminated with exception: [Rank 27] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200031 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f61d19deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f61d28a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f61d28a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f61d28a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f61b6001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f623047aaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f6230507c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f61d19deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7f61d287c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7f61d23a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7f61b6001db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7f623047aaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7f6230507c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +W1011 07:34:50.802000 24550 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 27981 closing signal SIGTERM +W1011 07:34:50.843000 24550 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 27985 closing signal SIGTERM +W1011 07:35:20.847000 24550 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 27981 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:24.645000 24550 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 27985 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +E1011 07:35:28.382000 24550 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -6) local_rank: 0 (pid: 27979) of binary: /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +Traceback (most recent call last): + File "/mnt/ssd/yulan/cache/installs/cpython-3.10.19-linux-x86_64-gnu/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/mnt/ssd/yulan/cache/installs/cpython-3.10.19-linux-x86_64-gnu/lib/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 905, in + main() + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 143, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 277, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-10-11_07:34:50 + host : cs-54361-84575-server + rank : 25 (local_rank: 1) + exitcode : -6 (pid: 27980) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 27980 +[2]: + time : 2026-10-11_07:34:50 + host : cs-54361-84575-server + rank : 27 (local_rank: 3) + exitcode : -6 (pid: 27982) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 27982 +[3]: + time : 2026-10-11_07:34:50 + host : cs-54361-84575-server + rank : 28 (local_rank: 4) + exitcode : -6 (pid: 27983) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 27983 +[4]: + time : 2026-10-11_07:34:50 + host : cs-54361-84575-server + rank : 29 (local_rank: 5) + exitcode : -6 (pid: 27984) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 27984 +[5]: + time : 2026-10-11_07:34:50 + host : cs-54361-84575-server + rank : 31 (local_rank: 7) + exitcode : -6 (pid: 27986) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 27986 +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-10-11_07:34:50 + host : cs-54361-84575-server + rank : 24 (local_rank: 0) + exitcode : -6 (pid: 27979) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 27979 +============================================================ diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node4.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node4.log new file mode 100644 index 0000000000000000000000000000000000000000..d129c694e634b1f96b0de85be3652607a0dc6c2b --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node4.log @@ -0,0 +1,784 @@ +W1011 02:40:46.623000 13141 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:46.623000 13141 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:46.956000 13143 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:46.956000 13143 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:47.321000 13154 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:47.321000 13154 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:47.479000 13137 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:47.479000 13137 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:47.552000 13146 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:47.552000 13146 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +W1011 02:40:47.582000 13152 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:47.582000 13152 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank 32 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank32]:[W1011 02:41:51.758818730 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank32]:[W1011 02:41:51.758834921 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 33 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank33]:[W1011 02:41:51.759233519 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank33]:[W1011 02:41:51.759244460 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 34 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank34]:[W1011 02:41:51.759661964 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank34]:[W1011 02:41:51.759674007 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 35 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank35]:[W1011 02:41:51.760016809 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank35]:[W1011 02:41:51.760132148 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 36 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank36]:[W1011 02:41:51.760573749 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank36]:[W1011 02:41:51.760590240 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 37 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank37]:[W1011 02:41:51.760987326 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank37]:[W1011 02:41:51.761002464 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 38 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank38]:[W1011 02:41:51.761286404 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank38]:[W1011 02:41:51.761302996 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 39 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank39]:[W1011 02:41:51.761816412 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank39]:[W1011 02:41:51.761828365 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank39]:[W1011 02:41:52.820672975 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 39] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank33]:[W1011 02:41:52.823434709 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 33] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank35]:[W1011 02:41:52.826499559 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 35] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank34]:[W1011 02:41:52.826975765 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 34] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank32]:[W1011 02:41:52.828371110 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 32] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank36]:[W1011 02:41:52.830769782 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 36] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank38]:[W1011 02:41:52.830977918 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 38] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank37]:[W1011 02:41:52.832558746 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 37] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank34]:[W1011 02:41:52.629592704 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank34]:[W1011 02:41:52.629623923 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank32]:[W1011 02:41:52.630434636 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank32]:[W1011 02:41:52.630471375 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank37]:[W1011 02:41:52.631229157 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank37]:[W1011 02:41:52.631260296 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank36]:[W1011 02:41:52.631417605 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank36]:[W1011 02:41:52.631443364 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank39]:[W1011 02:41:52.631482970 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank39]:[W1011 02:41:52.631507647 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank38]:[W1011 02:41:52.633026396 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank38]:[W1011 02:41:52.633058447 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank33]:[W1011 02:41:52.633093203 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank33]:[W1011 02:41:52.633131486 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank35]:[W1011 02:41:52.635200632 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank35]:[W1011 02:41:52.635219327 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +>>> start_samples: 0 +Initializing GlobIndexedDataset>>> start_samples: 0 +>>> start_samples: 0Initializing GlobIndexedDataset + + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + + + + + +>>> start_samples: 0>>> start_samples: 0 + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + + + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +W1011 04:40:56.498000 35962 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 04:40:56.498000 35962 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank37]:[E1011 07:33:47.210149084 ProcessGroupNCCL.cpp:685] [Rank 37] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200007 milliseconds before timing out. +[rank32]:[E1011 07:33:47.219530805 ProcessGroupNCCL.cpp:685] [Rank 32] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200026 milliseconds before timing out. +[rank37]:[E1011 07:33:47.222291225 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 37] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank32]:[E1011 07:33:47.222296104 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 32] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank37]:[E1011 07:33:47.223025111 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank32]:[E1011 07:33:47.223029749 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank36]:[E1011 07:33:47.226220427 ProcessGroupNCCL.cpp:685] [Rank 36] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200033 milliseconds before timing out. +[rank36]:[E1011 07:33:47.226456867 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 36] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank36]:[E1011 07:33:47.226586083 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank36]:[E1011 07:33:47.227066065 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 36] First PG on this rank to signal dumping. +[rank32]:[E1011 07:33:47.227054003 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 32] First PG on this rank to signal dumping. +[rank37]:[E1011 07:33:47.227091063 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 37] First PG on this rank to signal dumping. +[rank34]:[E1011 07:33:47.236324102 ProcessGroupNCCL.cpp:685] [Rank 34] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200043 milliseconds before timing out. +[rank34]:[E1011 07:33:47.236479327 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 34] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank34]:[E1011 07:33:47.236494055 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank34]:[E1011 07:33:47.236675180 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 34] First PG on this rank to signal dumping. +[rank38]:[E1011 07:33:47.251473132 ProcessGroupNCCL.cpp:685] [Rank 38] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200058 milliseconds before timing out. +[rank38]:[E1011 07:33:47.251621664 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 38] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank38]:[E1011 07:33:47.251638206 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank38]:[E1011 07:33:47.251821805 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 38] First PG on this rank to signal dumping. +[rank39]:[E1011 07:33:47.253593745 ProcessGroupNCCL.cpp:685] [Rank 39] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200061 milliseconds before timing out. +[rank39]:[E1011 07:33:47.253760021 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 39] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank39]:[E1011 07:33:47.253780630 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank39]:[E1011 07:33:47.253914885 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 39] First PG on this rank to signal dumping. +[rank33]:[E1011 07:33:47.288445314 ProcessGroupNCCL.cpp:685] [Rank 33] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200095 milliseconds before timing out. +[rank33]:[E1011 07:33:47.288617121 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 33] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank33]:[E1011 07:33:47.288727942 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank33]:[E1011 07:33:47.288943352 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 33] First PG on this rank to signal dumping. +[rank35]:[E1011 07:33:47.290613257 ProcessGroupNCCL.cpp:685] [Rank 35] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200097 milliseconds before timing out. +[rank35]:[E1011 07:33:47.290778041 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 35] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank35]:[E1011 07:33:47.290792488 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank35]:[E1011 07:33:47.290971288 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 35] First PG on this rank to signal dumping. +[rank37]:[E1011 07:33:47.529511144 ProcessGroupNCCL.cpp:685] [Rank 37] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200023 milliseconds before timing out. +[rank37]:[E1011 07:33:47.529610703 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 456(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank37]:[E1011 07:33:47.529640189 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank34]:[E1011 07:33:47.534921072 ProcessGroupNCCL.cpp:685] [Rank 34] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200035 milliseconds before timing out. +[rank34]:[E1011 07:33:47.535002227 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 456(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank34]:[E1011 07:33:47.535012336 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank33]:[E1011 07:33:47.554202050 ProcessGroupNCCL.cpp:685] [Rank 33] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200051 milliseconds before timing out. +[rank33]:[E1011 07:33:47.554306960 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 456(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank33]:[E1011 07:33:47.554319333 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank38]:[E1011 07:33:47.557773873 ProcessGroupNCCL.cpp:685] [Rank 38] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200054 milliseconds before timing out. +[rank38]:[E1011 07:33:47.557866729 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 456(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank38]:[E1011 07:33:47.557974715 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank32]:[E1011 07:33:47.583053428 ProcessGroupNCCL.cpp:685] [Rank 32] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200078 milliseconds before timing out. +[rank32]:[E1011 07:33:47.583145443 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 456(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank32]:[E1011 07:33:47.583156104 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank36]:[E1011 07:33:47.588076972 ProcessGroupNCCL.cpp:685] [Rank 36] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200082 milliseconds before timing out. +[rank36]:[E1011 07:33:47.588151293 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 456(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank36]:[E1011 07:33:47.588165871 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank39]:[E1011 07:33:47.593901519 ProcessGroupNCCL.cpp:685] [Rank 39] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200089 milliseconds before timing out. +[rank39]:[E1011 07:33:47.594005497 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 456(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank39]:[E1011 07:33:47.594018341 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank35]:[E1011 07:33:47.596155796 ProcessGroupNCCL.cpp:685] [Rank 35] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200088 milliseconds before timing out. +[rank35]:[E1011 07:33:47.596215750 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 456(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank35]:[E1011 07:33:47.596227793 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank38]:[W1011 07:33:47.558105604 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=99, addr=[cs-821eb-b87ce-server]:56744, remote=[10-239-252-55.cs-517b1-57d54-ssh-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fc31157eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fc2f55bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fc2f55bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fc2f55bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fc2f55b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fc2b4a95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fc2b4a9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fc2b4aa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fc2b4aa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fc298401db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fc31280daa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fc31289ac3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank34]:[W1011 07:33:47.537097070 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=99, addr=[cs-821eb-b87ce-server]:56712, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f49a457eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f49885bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f49885bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f49885bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f49885b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f4947a95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f4947a9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f4947aa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f4947aa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f492b401db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f49a57e4aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f49a5871c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank39]:[W1011 07:33:47.594148408 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=99, addr=[cs-821eb-b87ce-server]:56738, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ff1e297eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7ff1c69bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7ff1c69bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7ff1c69bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7ff1c69b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7ff185e95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7ff185e9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7ff185ea2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7ff185ea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7ff169801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7ff1e3bc3aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7ff1e3c50c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank33]:[W1011 07:33:47.554459340 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=99, addr=[cs-821eb-b87ce-server]:56728, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f369817eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f367c1bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f367c1bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f367c1bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f367c1b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f363b695880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f363b69599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f363b6a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f363b6a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f361f001db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f36993a2aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f369942fc3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank37]:[W1011 07:33:47.537103503 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=99, addr=[cs-821eb-b87ce-server]:56750, remote=[10-239-252-55.cs-517b1-57d54-ssh-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fee6acd9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fee4edbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fee4edbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fee4edbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fee4edb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fee0e295880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fee0e29599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fee0e2a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fee0e2a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fedf1c01db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fee6beeaaa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fee6bf77c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank35]:[W1011 07:33:47.596353722 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=99, addr=[cs-821eb-b87ce-server]:56766, remote=[10-239-252-55.cs-517b1-57d54-ssh-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f419997eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f417d9bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f417d9bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f417d9bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f417d9b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f413ce95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f413ce9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f413cea2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f413cea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f4120801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f419ab8baa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f419ac18c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank36]:[W1011 07:33:47.588366853 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=99, addr=[cs-821eb-b87ce-server]:56754, remote=[10-239-252-55.cs-517b1-57d54-ssh-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7feaa08d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fea849bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fea849bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fea849bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fea849b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fea43e95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fea43e9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fea43ea2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fea43ea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fea27801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7feaa1aecaa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7feaa1b79c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank32]:[W1011 07:33:47.583311179 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=99, addr=[cs-821eb-b87ce-server]:56756, remote=[10-239-252-55.cs-517b1-57d54-ssh-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f9c6a6d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f9c4e7bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f9c4e7bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f9c4e7bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f9c4e7b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f9c0dc95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f9c0dc9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f9c0dca2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f9c0dca4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f9bf1601db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f9c6b8d8aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f9c6b965c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank34]:[E1011 07:33:47.645099856 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 456(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f49a457eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f49885bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f49885bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f49885bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f49885b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f4947a95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f4947a9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f4947aa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f4947aa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f492b401db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f49a57e4aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f49a5871c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank32]:[E1011 07:33:47.645104104 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 456(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f9c6a6d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f9c4e7bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f9c4e7bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f9c4e7bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f9c4e7b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f9c0dc95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f9c0dc9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f9c0dca2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f9c0dca4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f9bf1601db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f9c6b8d8aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f9c6b965c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank33]:[E1011 07:33:47.645108723 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 456(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f369817eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f367c1bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f367c1bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f367c1bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f367c1b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f363b695880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f363b69599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f363b6a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f363b6a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f361f001db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f36993a2aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f369942fc3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank36]:[E1011 07:33:47.645115746 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 456(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7feaa08d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fea849bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fea849bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fea849bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fea849b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fea43e95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fea43e9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fea43ea2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fea43ea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fea27801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7feaa1aecaa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7feaa1b79c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank38]:[E1011 07:33:47.645116658 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 456(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fc31157eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fc2f55bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fc2f55bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fc2f55bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fc2f55b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fc2b4a95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fc2b4a9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fc2b4aa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fc2b4aa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fc298401db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fc31280daa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fc31289ac3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank35]:[E1011 07:33:47.645123110 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 456(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f419997eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f417d9bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f417d9bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f417d9bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f417d9b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f413ce95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f413ce9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f413cea2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f413cea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f4120801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f419ab8baa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f419ac18c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank39]:[E1011 07:33:47.645124403 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 456(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ff1e297eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7ff1c69bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7ff1c69bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7ff1c69bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7ff1c69b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7ff185e95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7ff185e9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7ff185ea2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7ff185ea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7ff169801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7ff1e3bc3aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7ff1e3c50c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank37]:[E1011 07:33:47.645135914 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 456(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fee6acd9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fee4edbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fee4edbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fee4edbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fee4edb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fee0e295880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fee0e29599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fee0e2a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fee0e2a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fedf1c01db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fee6beeaaa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fee6bf77c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank39]:[E1011 07:34:47.410360721 ProcessGroupNCCL.cpp:746] [Rank 39] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank39]:[E1011 07:34:47.410637147 ProcessGroupNCCL.cpp:760] [Rank 39] To avoid data inconsistency, we are taking the entire process down. +[rank39]:[E1011 07:34:47.414048163 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 39] Process group watchdog thread terminated with exception: [Rank 39] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200061 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ff1e297eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7ff185ea0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7ff185ea3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7ff185ea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7ff169801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7ff1e3bc3aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7ff1e3c50c3c in /lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 39] Process group watchdog thread terminated with exception: [Rank 39] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200061 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ff1e297eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7ff185ea0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7ff185ea3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7ff185ea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7ff169801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7ff1e3bc3aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7ff1e3c50c3c in /lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ff1e297eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7ff185e7c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7ff1859a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7ff169801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7ff1e3bc3aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7ff1e3c50c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank37]:[E1011 07:34:47.417907271 ProcessGroupNCCL.cpp:746] [Rank 37] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank37]:[E1011 07:34:47.417942638 ProcessGroupNCCL.cpp:760] [Rank 37] To avoid data inconsistency, we are taking the entire process down. +[rank37]:[E1011 07:34:47.418835706 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 37] Process group watchdog thread terminated with exception: [Rank 37] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200007 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fee6acd9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7fee0e2a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7fee0e2a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fee0e2a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7fedf1c01db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7fee6beeaaa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7fee6bf77c3c in /lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 37] Process group watchdog thread terminated with exception: [Rank 37] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200007 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fee6acd9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7fee0e2a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7fee0e2a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fee0e2a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7fedf1c01db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7fee6beeaaa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7fee6bf77c3c in /lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fee6acd9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7fee0e27c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7fee0dda68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7fedf1c01db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7fee6beeaaa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7fee6bf77c3c in /lib/x86_64-linux-gnu/libc.so.6) + +W1011 07:34:50.760000 20935 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 13137 closing signal SIGTERM +W1011 07:34:50.807000 20935 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 13141 closing signal SIGTERM +W1011 07:34:50.845000 20935 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 13143 closing signal SIGTERM +W1011 07:34:50.928000 20935 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 13146 closing signal SIGTERM +W1011 07:34:50.956000 20935 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 13149 closing signal SIGTERM +W1011 07:34:51.029000 20935 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 13154 closing signal SIGTERM +W1011 07:35:21.129000 20935 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 13137 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:24.339000 20935 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 13141 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:27.570000 20935 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 13143 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:31.355000 20935 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 13146 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:35.002000 20935 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 13149 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:38.550000 20935 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 13154 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +E1011 07:35:42.455000 20935 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -6) local_rank: 5 (pid: 13152) of binary: /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +Traceback (most recent call last): + File "/mnt/ssd/yulan/cache/installs/cpython-3.10.19-linux-x86_64-gnu/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/mnt/ssd/yulan/cache/installs/cpython-3.10.19-linux-x86_64-gnu/lib/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 905, in + main() + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 143, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 277, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-10-11_07:34:50 + host : cs-821eb-b87ce-server + rank : 39 (local_rank: 7) + exitcode : -6 (pid: 13157) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 13157 +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-10-11_07:34:50 + host : cs-821eb-b87ce-server + rank : 37 (local_rank: 5) + exitcode : -6 (pid: 13152) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 13152 +============================================================ diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node5.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node5.log new file mode 100644 index 0000000000000000000000000000000000000000..de3db7f35a81536118a36c09561074c0bcf2b45f --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node5.log @@ -0,0 +1,752 @@ +W1011 02:40:45.586000 11252 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:45.586000 11252 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:45.756000 11251 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:45.756000 11251 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:45.837000 11248 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:45.837000 11248 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:45.982000 11243 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:45.982000 11243 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:46.152000 11250 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:46.152000 11250 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +W1011 02:40:46.205000 11247 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:46.205000 11247 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:46.249000 11242 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:46.249000 11242 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:46.300000 11249 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:46.300000 11249 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank 41 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank41]:[W1011 02:41:50.505816228 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank41]:[W1011 02:41:50.505865341 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 40 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank40]:[W1011 02:41:50.510312093 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank40]:[W1011 02:41:50.510323688 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 43 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank43]:[W1011 02:41:50.511065813 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank43]:[W1011 02:41:50.511076267 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 46 is connected to 63[Gloo] Rank peer ranks. Expected number of connected peer ranks is : 6344 + is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 42 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank46]:[W1011 02:41:50.511700272 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[Gloo] Rank [rank46]:[W1011 02:41:50.511710919 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +45 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank44]:[W1011 02:41:50.511718968 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank42]:[W1011 02:41:50.511722667 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank44]:[W1011 02:41:50.511731511 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank42]:[W1011 02:41:50.511732102 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 47 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank45]:[W1011 02:41:50.511808002 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank45]:[W1011 02:41:50.511817953 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank47]:[W1011 02:41:50.511870858 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank47]:[W1011 02:41:50.511880705 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank [Gloo] Rank 22 is connected to is connected to 33 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 33 + +[Gloo] Rank [Gloo] Rank 22 is connected to is connected to 33 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 33 + +[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank47]:[W1011 02:41:51.545372795 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 47] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank41]:[W1011 02:41:51.546556958 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 41] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank40]:[W1011 02:41:51.549801121 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 40] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank46]:[W1011 02:41:52.558063429 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 46] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank43]:[W1011 02:41:52.559158296 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 43] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank45]:[W1011 02:41:52.559634764 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 45] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank42]:[W1011 02:41:52.559647567 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 42] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank44]:[W1011 02:41:52.560510980 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 44] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank41]:[W1011 02:41:52.333416473 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank41]:[W1011 02:41:52.333439449 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank47]:[W1011 02:41:52.333918301 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank47]:[W1011 02:41:52.333942021 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank44]:[W1011 02:41:52.333939059 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank44]:[W1011 02:41:52.333957880 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank40]:[W1011 02:41:52.334224908 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank40]:[W1011 02:41:52.334241617 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank45]:[W1011 02:41:52.334303216 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank45]:[W1011 02:41:52.334320138 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank42]:[W1011 02:41:52.334770853 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank42]:[W1011 02:41:52.334785452 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank46]:[W1011 02:41:52.334833690 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank46]:[W1011 02:41:52.334850410 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank43]:[W1011 02:41:52.334870678 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank43]:[W1011 02:41:52.334887733 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + + +>>> start_samples: 0 +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 + +Initializing GlobIndexedDataset +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 + + + +Initializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDataset + + + +>>> start_samples: 0 +Initializing GlobIndexedDataset +>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +W1011 04:40:53.648000 42253 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 04:40:53.648000 42253 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank42]:[E1011 07:33:47.941995374 ProcessGroupNCCL.cpp:685] [Rank 42] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200014 milliseconds before timing out. +[rank42]:[E1011 07:33:47.950712368 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 42] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank42]:[E1011 07:33:47.951331410 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank42]:[E1011 07:33:47.953801904 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 42] First PG on this rank to signal dumping. +[rank44]:[E1011 07:33:47.961744684 ProcessGroupNCCL.cpp:685] [Rank 44] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200043 milliseconds before timing out. +[rank44]:[E1011 07:33:47.961863045 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 44] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank44]:[E1011 07:33:47.961936942 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank44]:[E1011 07:33:47.962033517 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 44] First PG on this rank to signal dumping. +[rank41]:[E1011 07:33:47.973237292 ProcessGroupNCCL.cpp:685] [Rank 41] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200054 milliseconds before timing out. +[rank41]:[E1011 07:33:47.973337223 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 41] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank41]:[E1011 07:33:47.973352086 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank41]:[E1011 07:33:47.973429781 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 41] First PG on this rank to signal dumping. +[rank40]:[E1011 07:33:47.986961016 ProcessGroupNCCL.cpp:685] [Rank 40] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200068 milliseconds before timing out. +[rank40]:[E1011 07:33:47.987053817 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 40] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank40]:[E1011 07:33:47.987069218 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank40]:[E1011 07:33:47.987138171 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 40] First PG on this rank to signal dumping. +[rank43]:[E1011 07:33:47.994888779 ProcessGroupNCCL.cpp:685] [Rank 43] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200076 milliseconds before timing out. +[rank43]:[E1011 07:33:47.994992612 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 43] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank43]:[E1011 07:33:47.995014069 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank43]:[E1011 07:33:47.995086122 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 43] First PG on this rank to signal dumping. +[rank46]:[E1011 07:33:47.013346419 ProcessGroupNCCL.cpp:685] [Rank 46] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200094 milliseconds before timing out. +[rank46]:[E1011 07:33:47.013439901 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 46] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank46]:[E1011 07:33:47.013451001 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank46]:[E1011 07:33:47.013516367 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 46] First PG on this rank to signal dumping. +[rank47]:[E1011 07:33:47.015347518 ProcessGroupNCCL.cpp:685] [Rank 47] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200096 milliseconds before timing out. +[rank47]:[E1011 07:33:47.015437214 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 47] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank47]:[E1011 07:33:47.015452259 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank47]:[E1011 07:33:47.015520957 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 47] First PG on this rank to signal dumping. +[rank45]:[E1011 07:33:47.016934207 ProcessGroupNCCL.cpp:685] [Rank 45] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200098 milliseconds before timing out. +[rank45]:[E1011 07:33:47.017037295 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 45] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank45]:[E1011 07:33:47.017051875 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank45]:[E1011 07:33:47.017124746 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 45] First PG on this rank to signal dumping. +[rank42]:[E1011 07:33:47.233501231 ProcessGroupNCCL.cpp:685] [Rank 42] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200000 milliseconds before timing out. +[rank42]:[E1011 07:33:47.233540621 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 457(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank42]:[E1011 07:33:47.233547228 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank44]:[E1011 07:33:47.261822238 ProcessGroupNCCL.cpp:685] [Rank 44] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200030 milliseconds before timing out. +[rank44]:[E1011 07:33:47.261859218 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 457(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank44]:[E1011 07:33:47.261866613 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank40]:[E1011 07:33:47.290844558 ProcessGroupNCCL.cpp:685] [Rank 40] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200058 milliseconds before timing out. +[rank40]:[E1011 07:33:47.290881868 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 457(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank40]:[E1011 07:33:47.290888842 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank45]:[E1011 07:33:47.292160026 ProcessGroupNCCL.cpp:685] [Rank 45] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200057 milliseconds before timing out. +[rank45]:[E1011 07:33:47.292198735 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 457(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank45]:[E1011 07:33:47.292208614 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank43]:[E1011 07:33:47.295155143 ProcessGroupNCCL.cpp:685] [Rank 43] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200068 milliseconds before timing out. +[rank43]:[E1011 07:33:47.295196283 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 457(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank43]:[E1011 07:33:47.295204839 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank41]:[E1011 07:33:47.296423232 ProcessGroupNCCL.cpp:685] [Rank 41] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200066 milliseconds before timing out. +[rank41]:[E1011 07:33:47.296460382 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 457(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank41]:[E1011 07:33:47.296469585 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank46]:[E1011 07:33:47.313701998 ProcessGroupNCCL.cpp:685] [Rank 46] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200083 milliseconds before timing out. +[rank46]:[E1011 07:33:47.313742694 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 457(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank46]:[E1011 07:33:47.313752296 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank47]:[E1011 07:33:47.317287834 ProcessGroupNCCL.cpp:685] [Rank 47] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200086 milliseconds before timing out. +[rank47]:[E1011 07:33:47.317326673 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 457(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank47]:[E1011 07:33:47.317335117 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank41]:[W1011 07:33:47.296552832 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=95, addr=[cs-78847-8a49c-server]:56474, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f0669bdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f06ab5bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f06ab5bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f06ab5bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f06ab5b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f066aa95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f066aa9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f066aaa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f066aaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f064e201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f06c8688aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f06c8715c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank43]:[W1011 07:33:47.295283566 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=95, addr=[cs-78847-8a49c-server]:56462, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f087f37eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f08633bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f08633bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f08633bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f08633b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f0822895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f082289599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f08228a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f08228a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f0806201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f08805a3aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f0880630c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank40]:[W1011 07:33:47.290997249 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=95, addr=[cs-78847-8a49c-server]:56482, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ff3e097eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7ff3c49bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7ff3c49bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7ff3c49bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7ff3c49b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7ff383e95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7ff383e9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7ff383ea2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7ff383ea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7ff367801db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7ff3e1bd6aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7ff3e1c63c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank46]:[W1011 07:33:47.313823982 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=95, addr=[cs-78847-8a49c-server]:56450, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7faac3bdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fab055bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fab055bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fab055bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fab055b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7faac4a95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7faac4a9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7faac4aa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7faac4aa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7faaa8201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fab2265faa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fab226ecc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank42]:[W1011 07:33:47.240633063 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=95, addr=[cs-78847-8a49c-server]:56492, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f840497eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f83e89bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f83e89bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f83e89bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f83e89b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f83a7e95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f83a7e9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f83a7ea2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f83a7ea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f838b801db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f8405c0aaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f8405c97c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank45]:[W1011 07:33:47.292280265 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=95, addr=[cs-78847-8a49c-server]:56500, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbe2137eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fbe053bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fbe053bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fbe053bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fbe053b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fbdc4895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fbdc489599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fbdc48a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fbdc48a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fbda8201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fbe22589aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fbe22616c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank47]:[W1011 07:33:47.317402031 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=95, addr=[cs-78847-8a49c-server]:56472, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f6f4857eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f6f2c5bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f6f2c5bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f6f2c5bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f6f2c5b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f6eeba95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f6eeba9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f6eebaa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f6eebaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f6ecf401db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f6f49785aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f6f49812c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank44]:[W1011 07:33:47.261942636 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=95, addr=[cs-78847-8a49c-server]:56512, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fdd4a57eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fdd2e5bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fdd2e5bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fdd2e5bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fdd2e5b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fdceda95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fdceda9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fdcedaa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fdcedaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fdcd1401db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fdd4b7e1aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fdd4b86ec3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank41]:[E1011 07:33:47.343596122 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 457(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f0669bdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f06ab5bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f06ab5bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f06ab5bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f06ab5b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f066aa95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f066aa9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f066aaa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f066aaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f064e201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f06c8688aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f06c8715c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank43]:[E1011 07:33:47.343600008 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 457(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f087f37eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f08633bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f08633bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f08633bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f08633b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f0822895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f082289599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f08228a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f08228a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f0806201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f08805a3aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f0880630c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank45]:[E1011 07:33:47.343604283 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 457(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fbe2137eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fbe053bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fbe053bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fbe053bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fbe053b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fbdc4895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fbdc489599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fbdc48a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fbdc48a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fbda8201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fbe22589aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fbe22616c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank44]:[E1011 07:33:47.343607927 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 457(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fdd4a57eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fdd2e5bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fdd2e5bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fdd2e5bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fdd2e5b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fdceda95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fdceda9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fdcedaa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fdcedaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fdcd1401db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fdd4b7e1aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fdd4b86ec3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank46]:[E1011 07:33:47.343613809 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 457(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7faac3bdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fab055bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fab055bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fab055bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fab055b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7faac4a95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7faac4a9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7faac4aa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7faac4aa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7faaa8201db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fab2265faa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fab226ecc3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank47]:[E1011 07:33:47.343617338 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 457(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f6f4857eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f6f2c5bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f6f2c5bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f6f2c5bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f6f2c5b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f6eeba95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f6eeba9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f6eebaa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f6eebaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f6ecf401db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f6f49785aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f6f49812c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank40]:[E1011 07:33:47.343617158 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 457(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ff3e097eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7ff3c49bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7ff3c49bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7ff3c49bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7ff3c49b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7ff383e95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7ff383e9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7ff383ea2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7ff383ea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7ff367801db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7ff3e1bd6aa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7ff3e1c63c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank42]:[E1011 07:33:47.343619067 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 457(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f840497eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f83e89bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f83e89bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f83e89bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f83e89b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f83a7e95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f83a7e9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f83a7ea2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f83a7ea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f838b801db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f8405c0aaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f8405c97c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +[rank42]:[E1011 07:34:47.076265659 ProcessGroupNCCL.cpp:746] [Rank 42] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank42]:[E1011 07:34:47.076315624 ProcessGroupNCCL.cpp:760] [Rank 42] To avoid data inconsistency, we are taking the entire process down. +[rank42]:[E1011 07:34:47.079508511 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 42] Process group watchdog thread terminated with exception: [Rank 42] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200014 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f840497eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f83a7ea0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f83a7ea3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f83a7ea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f838b801db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f8405c0aaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f8405c97c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 42] Process group watchdog thread terminated with exception: [Rank 42] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200014 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f840497eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f83a7ea0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f83a7ea3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f83a7ea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f838b801db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f8405c0aaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f8405c97c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f840497eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7f83a7e7c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7f83a79a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7f838b801db4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7f8405c0aaa4 in /usr/lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7f8405c97c3c in /usr/lib/x86_64-linux-gnu/libc.so.6) + +W1011 07:34:51.966000 9435 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 11242 closing signal SIGTERM +W1011 07:34:51.988000 9435 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 11243 closing signal SIGTERM +W1011 07:34:51.992000 9435 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 11248 closing signal SIGTERM +W1011 07:34:52.012000 9435 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 11249 closing signal SIGTERM +W1011 07:34:52.024000 9435 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 11250 closing signal SIGTERM +W1011 07:34:52.040000 9435 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 11251 closing signal SIGTERM +W1011 07:34:52.060000 9435 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 11252 closing signal SIGTERM +W1011 07:35:22.065000 9435 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 11242 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:24.907000 9435 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 11243 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:27.777000 9435 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 11248 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:30.897000 9435 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 11249 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:33.750000 9435 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 11250 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:36.719000 9435 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 11251 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:39.621000 9435 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 11252 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +E1011 07:35:42.713000 9435 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -6) local_rank: 2 (pid: 11247) of binary: /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +Traceback (most recent call last): + File "/mnt/ssd/yulan/cache/installs/cpython-3.10.19-linux-x86_64-gnu/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/mnt/ssd/yulan/cache/installs/cpython-3.10.19-linux-x86_64-gnu/lib/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 905, in + main() + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 143, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 277, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-10-11_07:34:51 + host : cs-78847-8a49c-server + rank : 42 (local_rank: 2) + exitcode : -6 (pid: 11247) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 11247 +============================================================ diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node6.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node6.log new file mode 100644 index 0000000000000000000000000000000000000000..bc5f8658c0d4288beb10d4d95f06ce634e36ce25 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node6.log @@ -0,0 +1,784 @@ +W1011 02:40:46.569000 44881 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:46.569000 44881 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:46.675000 44882 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:46.675000 44882 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:46.738000 44885 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:46.738000 44885 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +W1011 02:40:46.792000 44887 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:46.792000 44887 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:47.336000 44883 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:47.336000 44883 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:47.450000 44886 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:47.450000 44886 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank 49 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank49]:[W1011 02:41:50.053716792 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank49]:[W1011 02:41:50.053733734 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 50 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 55 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank50]:[W1011 02:41:51.040599829 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank50]:[W1011 02:41:51.040621199 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank55]:[W1011 02:41:51.040721599 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank55]:[W1011 02:41:51.040738040 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 48 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank48]:[W1011 02:41:51.042131862 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank48]:[W1011 02:41:51.042146650 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 53 is connected to 63[Gloo] Rank peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 526354 is connected to + is connected to 6363 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 6363 + +[Gloo] Rank 51 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank53]:[W1011 02:41:51.043473255 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank54]:[W1011 02:41:51.043475650 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank51]:[W1011 02:41:51.043473265 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank52]:[W1011 02:41:51.043476782 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank53]:[W1011 02:41:51.043488705 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank54]:[W1011 02:41:51.043489476 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank51]:[W1011 02:41:51.043492502 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank52]:[W1011 02:41:51.043496108 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank49]:[W1011 02:41:52.097089302 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 49] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank55]:[W1011 02:41:52.106373526 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 55] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank48]:[W1011 02:41:52.108129733 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 48] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank51]:[W1011 02:41:52.112840480 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 51] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank50]:[W1011 02:41:52.113626033 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 50] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank53]:[W1011 02:41:52.114583351 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 53] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank54]:[W1011 02:41:52.115540288 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 54] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank52]:[W1011 02:41:52.116491174 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 52] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank51]:[W1011 02:41:52.908072970 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank51]:[W1011 02:41:52.908107896 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank52]:[W1011 02:41:52.909140326 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank52]:[W1011 02:41:52.909179099 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank55]:[W1011 02:41:52.910640760 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank55]:[W1011 02:41:52.910675205 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank49]:[W1011 02:41:52.911012933 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank49]:[W1011 02:41:52.911048991 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank48]:[W1011 02:41:52.911211378 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank48]:[W1011 02:41:52.911229301 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank54]:[W1011 02:41:52.911780693 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank54]:[W1011 02:41:52.911797354 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank53]:[W1011 02:41:52.912501464 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank53]:[W1011 02:41:52.912517113 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank50]:[W1011 02:41:52.914823609 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank50]:[W1011 02:41:52.914862743 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + + + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDataset +>>> start_samples: 0 +Initializing GlobIndexedDataset +>>> start_samples: 0Initializing GlobIndexedDataset + +>>> start_samples: 0>>> start_samples: 0 + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0 +Initializing GlobIndexedDataset>>> start_samples: 0 +>>> start_samples: 0 + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +W1011 04:40:56.485000 18211 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 04:40:56.485000 18211 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank49]:[E1011 07:33:52.126247554 ProcessGroupNCCL.cpp:685] [Rank 49] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200005 milliseconds before timing out. +[rank55]:[E1011 07:33:52.139083632 ProcessGroupNCCL.cpp:685] [Rank 55] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200026 milliseconds before timing out. +[rank49]:[E1011 07:33:52.139418791 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 49] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank55]:[E1011 07:33:52.139430282 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 55] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank49]:[E1011 07:33:52.139691974 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank55]:[E1011 07:33:52.139697764 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank49]:[E1011 07:33:52.144251122 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 49] First PG on this rank to signal dumping. +[rank55]:[E1011 07:33:52.144256312 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 55] First PG on this rank to signal dumping. +[rank51]:[E1011 07:33:52.166336944 ProcessGroupNCCL.cpp:685] [Rank 51] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200053 milliseconds before timing out. +[rank51]:[E1011 07:33:52.166512874 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 51] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank51]:[E1011 07:33:52.166537119 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank51]:[E1011 07:33:52.166686450 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 51] First PG on this rank to signal dumping. +[rank52]:[E1011 07:33:52.173199697 ProcessGroupNCCL.cpp:685] [Rank 52] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200060 milliseconds before timing out. +[rank52]:[E1011 07:33:52.173374966 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 52] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank52]:[E1011 07:33:52.173401927 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank52]:[E1011 07:33:52.173662757 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 52] First PG on this rank to signal dumping. +[rank53]:[E1011 07:33:52.188813690 ProcessGroupNCCL.cpp:685] [Rank 53] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200075 milliseconds before timing out. +[rank53]:[E1011 07:33:52.188972528 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 53] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank53]:[E1011 07:33:52.188986053 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank53]:[E1011 07:33:52.189165991 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 53] First PG on this rank to signal dumping. +[rank54]:[E1011 07:33:52.190993923 ProcessGroupNCCL.cpp:685] [Rank 54] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200078 milliseconds before timing out. +[rank54]:[E1011 07:33:52.191189520 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 54] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank54]:[E1011 07:33:52.191212974 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank54]:[E1011 07:33:52.191386640 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 54] First PG on this rank to signal dumping. +[rank48]:[E1011 07:33:52.198818321 ProcessGroupNCCL.cpp:685] [Rank 48] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200085 milliseconds before timing out. +[rank48]:[E1011 07:33:52.199033696 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 48] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank48]:[E1011 07:33:52.199175602 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank48]:[E1011 07:33:52.199360830 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 48] First PG on this rank to signal dumping. +[rank50]:[E1011 07:33:52.208735138 ProcessGroupNCCL.cpp:685] [Rank 50] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200095 milliseconds before timing out. +[rank50]:[E1011 07:33:52.208950733 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 50] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank50]:[E1011 07:33:52.208968336 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank50]:[E1011 07:33:52.209108790 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 50] First PG on this rank to signal dumping. +[rank54]:[E1011 07:33:52.476996085 ProcessGroupNCCL.cpp:685] [Rank 54] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200021 milliseconds before timing out. +[rank54]:[E1011 07:33:52.477091594 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 458(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank54]:[E1011 07:33:52.477106432 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank48]:[E1011 07:33:52.500354726 ProcessGroupNCCL.cpp:685] [Rank 48] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200060 milliseconds before timing out. +[rank48]:[E1011 07:33:52.500426381 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 458(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank48]:[E1011 07:33:52.500436650 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank50]:[E1011 07:33:52.511830690 ProcessGroupNCCL.cpp:685] [Rank 50] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200069 milliseconds before timing out. +[rank50]:[E1011 07:33:52.511925137 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 458(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank50]:[E1011 07:33:52.511937250 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank49]:[E1011 07:33:52.514828187 ProcessGroupNCCL.cpp:685] [Rank 49] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200072 milliseconds before timing out. +[rank49]:[E1011 07:33:52.514870106 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 458(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank49]:[E1011 07:33:52.514920170 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank53]:[E1011 07:33:52.523635942 ProcessGroupNCCL.cpp:685] [Rank 53] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200078 milliseconds before timing out. +[rank53]:[E1011 07:33:52.523718728 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 458(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank53]:[E1011 07:33:52.523734177 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank51]:[E1011 07:33:52.529513576 ProcessGroupNCCL.cpp:685] [Rank 51] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200087 milliseconds before timing out. +[rank51]:[E1011 07:33:52.529611720 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 458(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank51]:[E1011 07:33:52.529623022 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank52]:[E1011 07:33:52.539079273 ProcessGroupNCCL.cpp:685] [Rank 52] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200091 milliseconds before timing out. +[rank52]:[E1011 07:33:52.539129819 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 458(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank52]:[E1011 07:33:52.539138805 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank55]:[E1011 07:33:52.539386961 ProcessGroupNCCL.cpp:685] [Rank 55] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200096 milliseconds before timing out. +[rank55]:[E1011 07:33:52.539501456 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 458(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank55]:[E1011 07:33:52.539516665 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank51]:[W1011 07:33:52.529754919 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=100, addr=[cs-443fc-07dce-server]:49838, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f1fd32d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f1fb73bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f1fb73bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f1fb73bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f1fb73b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f1f76895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f1f7689599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f1f768a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f1f768a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f1f5a201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f1fd44c2aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f1fd454fc3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank52]:[W1011 07:33:52.539224156 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=100, addr=[cs-443fc-07dce-server]:49790, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7effcf37eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7effb33bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7effb33bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7effb33bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7effb33b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7eff72895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7eff7289599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7eff728a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7eff728a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7eff56201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7effd0557aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7effd05e4c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank54]:[W1011 07:33:52.486992381 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=100, addr=[cs-443fc-07dce-server]:49820, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f3512b7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f34f6bbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f34f6bbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f34f6bbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f34f6bb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f34b6095880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f34b609599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f34b60a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f34b60a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f3499a01db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f3513d91aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f3513e1ec3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank48]:[W1011 07:33:52.500540545 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=100, addr=[cs-443fc-07dce-server]:49822, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f1ce01deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f1d21bbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f1d21bbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f1d21bbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f1d21bb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f1ce1095880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f1ce109599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f1ce10a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f1ce10a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f1cc4801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f1d3ec91aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f1d3ed1ec3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank55]:[W1011 07:33:52.539696302 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=100, addr=[cs-443fc-07dce-server]:49800, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f241dbdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f245f5bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f245f5bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f245f5bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f245f5b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f241ea95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f241ea9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f241eaa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f241eaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f2402201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f247c651aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f247c6dec3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank50]:[W1011 07:33:52.512063717 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=100, addr=[cs-443fc-07dce-server]:49818, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f33c677eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f33aa7bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f33aa7bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f33aa7bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f33aa7b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f3369c95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f3369c9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f3369ca2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f3369ca4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f334d601db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f33c79c3aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f33c7a50c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank49]:[W1011 07:33:52.515035156 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=100, addr=[cs-443fc-07dce-server]:49812, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fa51f97eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fa5039bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fa5039bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fa5039bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fa5039b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fa4c2e95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fa4c2e9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fa4c2ea2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fa4c2ea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fa4a6801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fa520c3aaa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fa520cc7c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank53]:[W1011 07:33:52.523871735 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=100, addr=[cs-443fc-07dce-server]:49806, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f2b3c57eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f2b205bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f2b205bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f2b205bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f2b205b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f2adfa95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f2adfa9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f2adfaa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f2adfaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f2ac3401db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f2b3d7b8aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f2b3d845c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank48]:[E1011 07:33:52.606058645 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 458(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f1ce01deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f1d21bbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f1d21bbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f1d21bbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f1d21bb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f1ce1095880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f1ce109599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f1ce10a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f1ce10a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f1cc4801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f1d3ec91aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f1d3ed1ec3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank55]:[E1011 07:33:52.606066890 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 458(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f241dbdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f245f5bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f245f5bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f245f5bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f245f5b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f241ea95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f241ea9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f241eaa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f241eaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f2402201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f247c651aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f247c6dec3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank53]:[E1011 07:33:52.606069866 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 458(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f2b3c57eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f2b205bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f2b205bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f2b205bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f2b205b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f2adfa95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f2adfa9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f2adfaa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f2adfaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f2ac3401db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f2b3d7b8aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f2b3d845c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank54]:[E1011 07:33:52.606080987 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 458(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f3512b7eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f34f6bbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f34f6bbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f34f6bbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f34f6bb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f34b6095880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f34b609599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f34b60a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f34b60a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f3499a01db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f3513d91aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f3513e1ec3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank49]:[E1011 07:33:52.606088070 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 458(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fa51f97eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fa5039bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fa5039bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fa5039bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fa5039b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fa4c2e95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fa4c2e9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fa4c2ea2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fa4c2ea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fa4a6801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fa520c3aaa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fa520cc7c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank52]:[E1011 07:33:52.606100212 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 458(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7effcf37eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7effb33bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7effb33bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7effb33bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7effb33b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7eff72895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7eff7289599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7eff728a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7eff728a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7eff56201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7effd0557aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7effd05e4c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank50]:[E1011 07:33:52.606105302 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 458(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f33c677eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f33aa7bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f33aa7bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f33aa7bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f33aa7b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f3369c95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f3369c9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f3369ca2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f3369ca4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f334d601db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f33c79c3aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f33c7a50c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank51]:[E1011 07:33:52.606108388 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 458(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f1fd32d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f1fb73bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f1fb73bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f1fb73bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f1fb73b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f1f76895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f1f7689599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f1f768a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f1f768a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f1f5a201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f1fd44c2aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f1fd454fc3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank55]:[E1011 07:34:52.319041269 ProcessGroupNCCL.cpp:746] [Rank 55] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank55]:[E1011 07:34:52.319110680 ProcessGroupNCCL.cpp:760] [Rank 55] To avoid data inconsistency, we are taking the entire process down. +[rank55]:[E1011 07:34:52.322401096 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 55] Process group watchdog thread terminated with exception: [Rank 55] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200026 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f241dbdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f241eaa0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f241eaa3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f241eaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f2402201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f247c651aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f247c6dec3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank52]:[E1011 07:34:52.322706931 ProcessGroupNCCL.cpp:746] [Rank 52] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank52]:[E1011 07:34:52.322742267 ProcessGroupNCCL.cpp:760] [Rank 52] To avoid data inconsistency, we are taking the entire process down. +terminate called after throwing an instance of 'c10::DistBackendError' +[rank52]:[E1011 07:34:52.323581004 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 52] Process group watchdog thread terminated with exception: [Rank 52] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200060 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7effcf37eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7eff728a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7eff728a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7eff728a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7eff56201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7effd0557aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7effd05e4c3c in /lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 55] Process group watchdog thread terminated with exception: [Rank 55] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200026 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f241dbdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f241eaa0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f241eaa3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f241eaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f2402201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f247c651aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f247c6dec3c in /lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f241dbdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7f241ea7c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7f241e5a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7f2402201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7f247c651aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7f247c6dec3c in /lib/x86_64-linux-gnu/libc.so.6) + + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 52] Process group watchdog thread terminated with exception: [Rank 52] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200060 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7effcf37eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7eff728a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7eff728a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7eff728a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7eff56201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7effd0557aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7effd05e4c3c in /lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7effcf37eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7eff7287c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7eff723a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7eff56201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7effd0557aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7effd05e4c3c in /lib/x86_64-linux-gnu/libc.so.6) + +W1011 07:34:53.007000 43802 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44880 closing signal SIGTERM +W1011 07:34:53.035000 43802 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44881 closing signal SIGTERM +W1011 07:34:53.058000 43802 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44882 closing signal SIGTERM +W1011 07:34:53.121000 43802 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44883 closing signal SIGTERM +W1011 07:34:53.136000 43802 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44885 closing signal SIGTERM +W1011 07:34:53.229000 43802 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44886 closing signal SIGTERM +W1011 07:35:23.237000 43802 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 44880 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:26.284000 43802 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 44881 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:29.497000 43802 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 44882 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:32.696000 43802 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 44883 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:35.730000 43802 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 44885 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:38.632000 43802 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 44886 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +E1011 07:35:41.982000 43802 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -6) local_rank: 4 (pid: 44884) of binary: /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +Traceback (most recent call last): + File "/mnt/ssd/yulan/cache/installs/cpython-3.10.19-linux-x86_64-gnu/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/mnt/ssd/yulan/cache/installs/cpython-3.10.19-linux-x86_64-gnu/lib/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 905, in + main() + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 143, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 277, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-10-11_07:34:53 + host : cs-443fc-07dce-server + rank : 55 (local_rank: 7) + exitcode : -6 (pid: 44887) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 44887 +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-10-11_07:34:53 + host : cs-443fc-07dce-server + rank : 52 (local_rank: 4) + exitcode : -6 (pid: 44884) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 44884 +============================================================ diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node7.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node7.log new file mode 100644 index 0000000000000000000000000000000000000000..be7ff9df4c727a5dc8dc4533dccd7b1902f5aba0 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/node7.log @@ -0,0 +1,5096 @@ +W1011 02:40:39.415000 470430 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:39.415000 470430 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:39.454000 470433 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:39.454000 470433 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:39.467000 470432 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:39.467000 470432 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:39.517000 470431 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:39.517000 470431 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:39.552000 470434 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:39.552000 470434 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:39.571000 470435 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:39.571000 470435 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +W1011 02:40:39.766000 470429 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 02:40:39.766000 470429 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +WARNING: TensorBoard writing requested but is not available (are you using PyTorch 1.1.0 or later?), no TensorBoard logs will be written. +wandb: Currently logged in as: huyiwen to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run df4j15aa +wandb: Tracking run with wandb version 0.23.0 +wandb: Run data is saved locally in /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_024051-df4j15aa +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run stage15_115B_8node_gdnlralpha_off_20261003a +wandb: ⭐️ View project at https://wandb.ai/huyiwen/qwen2.5_3b_mom_cpt +wandb: 🚀 View run at https://wandb.ai/huyiwen/qwen2.5_3b_mom_cpt/runs/df4j15aa +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank [Gloo] Rank 56 is connected to 57[Gloo] Rank 63 is connected to peer ranks. [Gloo] Rank 5863[Gloo] Rank Expected number of connected peer ranks is : is connected to [Gloo] Rank 59 peer ranks. [Gloo] Rank 636063[Gloo] Rank 61 is connected to Expected number of connected peer ranks is : +63 is connected to peer ranks. is connected to 626363 is connected to 63Expected number of connected peer ranks is : 63 is connected to peer ranks. +63 peer ranks. 63 peer ranks. 63Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : +Expected number of connected peer ranks is : peer ranks. 63Expected number of connected peer ranks is : 6363Expected number of connected peer ranks is : +63 + +63 + +[rank56]:[W1011 02:41:50.521072997 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank58]:[W1011 02:41:50.521074754 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank56]:[W1011 02:41:50.521084989 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank58]:[W1011 02:41:50.521085749 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank63]:[W1011 02:41:50.521090459 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank63]:[W1011 02:41:50.521100673 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank57]:[W1011 02:41:50.521097388 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank57]:[W1011 02:41:50.521109029 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank61]:[W1011 02:41:50.521105984 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank61]:[W1011 02:41:50.521116659 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank59]:[W1011 02:41:50.521118597 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank62]:[W1011 02:41:50.521123552 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank60]:[W1011 02:41:50.521124357 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank59]:[W1011 02:41:50.521131188 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank62]:[W1011 02:41:50.521133508 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank60]:[W1011 02:41:50.521135384 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 3[Gloo] Rank is connected to 3 peer ranks. 3[Gloo] Rank Expected number of connected peer ranks is : is connected to 33[Gloo] Rank 3 is connected to +3 peer ranks. 3 is connected to Expected number of connected peer ranks is : peer ranks. 33Expected number of connected peer ranks is : peer ranks. +3Expected number of connected peer ranks is : +3 +[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank57]:[W1011 02:41:51.549766378 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 57] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank63]:[W1011 02:41:51.554893043 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 63] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank56]:[W1011 02:41:51.555035693 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 56] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank59]:[W1011 02:41:51.555981063 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 59] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank58]:[W1011 02:41:51.556586354 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 58] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank62]:[W1011 02:41:51.557535290 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 62] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank61]:[W1011 02:41:51.559509434 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 61] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank60]:[W1011 02:41:51.559937326 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 60] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank60]:[W1011 02:41:52.336420415 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank60]:[W1011 02:41:52.336443740 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank57]:[W1011 02:41:52.336961265 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank57]:[W1011 02:41:52.336979541 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank61]:[W1011 02:41:52.337017203 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank61]:[W1011 02:41:52.337038061 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank56]:[W1011 02:41:52.337036623 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank56]:[W1011 02:41:52.337055703 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank59]:[W1011 02:41:52.337260352 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank59]:[W1011 02:41:52.337276466 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank62]:[W1011 02:41:52.337275251 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank62]:[W1011 02:41:52.337289152 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank58]:[W1011 02:41:52.337632470 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank58]:[W1011 02:41:52.337648273 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank63]:[W1011 02:41:52.337671740 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank63]:[W1011 02:41:52.337706215 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable + failed to find checkpoint /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints/iter_0006855 in wandb +times across ranks (ms): + load-checkpoint: + rank 0: 70676.94 + rank 1: 70676.98 + rank 2: 70677.02 + rank 3: 70677.03 + rank 4: 70676.93 + rank 5: 70676.98 + rank 6: 70676.96 + rank 7: 70677.20 + rank 8: 70676.56 + rank 9: 70676.51 + rank 10: 70676.51 + rank 11: 70676.51 + rank 12: 70676.60 + rank 13: 70676.54 + rank 14: 70676.54 + rank 15: 70676.59 + rank 16: 70676.02 + rank 17: 70676.02 + rank 18: 70676.05 + rank 19: 70675.91 + rank 20: 70676.02 + rank 21: 70676.15 + rank 22: 70676.09 + rank 23: 70676.02 + rank 24: 70677.85 + rank 25: 70677.41 + rank 26: 70677.81 + rank 27: 70677.70 + rank 28: 70677.40 + rank 29: 70677.56 + rank 30: 70677.66 + rank 31: 70677.90 + rank 32: 70676.73 + rank 33: 70676.55 + rank 34: 70676.73 + rank 35: 70676.72 + rank 36: 70676.57 + rank 37: 70676.41 + rank 38: 70676.45 + rank 39: 70676.42 + rank 40: 70677.23 + rank 41: 70677.12 + rank 42: 70677.20 + rank 43: 70677.09 + rank 44: 70677.28 + rank 45: 70677.34 + rank 46: 70677.25 + rank 47: 70677.23 + rank 48: 70676.09 + rank 49: 70676.16 + rank 50: 70676.20 + rank 51: 70676.14 + rank 52: 70676.09 + rank 53: 70676.17 + rank 54: 70676.09 + rank 55: 70676.10 + rank 56: 70676.09 + rank 57: 70676.02 + rank 58: 70676.00 + rank 59: 70676.00 + rank 60: 70676.11 + rank 61: 70676.09 + rank 62: 70675.99 + rank 63: 70675.90 +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset>>> start_samples: 0 + +>>> start_samples: 0>>> start_samples: 0 + +Initializing GlobIndexedDataset>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDataset + +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + +pad_document_marker: 12051293pad_document_marker: 12051293pad_document_marker: 12051293 + + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True + + +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + + + +Initializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDataset + + +>>> start_samples: 0Initializing GlobIndexedDataset>>> start_samples: 0 + +>>> start_samples: 0 + +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +times across ranks (ms): + model-and-optimizer-setup: + rank 0: 96676.52 + rank 1: 96663.08 + rank 2: 96661.59 + rank 3: 96660.48 + rank 4: 96659.46 + rank 5: 96663.52 + rank 6: 96661.37 + rank 7: 96662.91 + rank 8: 96655.41 + rank 9: 96656.22 + rank 10: 96656.09 + rank 11: 96656.05 + rank 12: 96655.69 + rank 13: 96656.28 + rank 14: 96656.61 + rank 15: 96656.35 + rank 16: 96660.38 + rank 17: 96661.19 + rank 18: 96661.30 + rank 19: 96660.59 + rank 20: 96661.22 + rank 21: 96660.96 + rank 22: 96660.71 + rank 23: 96659.78 + rank 24: 96667.94 + rank 25: 96666.74 + rank 26: 96670.86 + rank 27: 96673.66 + rank 28: 96670.18 + rank 29: 96668.62 + rank 30: 96669.54 + rank 31: 96668.02 + rank 32: 96663.48 + rank 33: 96664.23 + rank 34: 96664.10 + rank 35: 96663.91 + rank 36: 96662.91 + rank 37: 96664.40 + rank 38: 96663.92 + rank 39: 96663.67 + rank 40: 96676.53 + rank 41: 96676.30 + rank 42: 96675.44 + rank 43: 96676.32 + rank 44: 96676.50 + rank 45: 96676.29 + rank 46: 96676.12 + rank 47: 96676.02 + rank 48: 96661.52 + rank 49: 96661.59 + rank 50: 96661.21 + rank 51: 96661.23 + rank 52: 96660.66 + rank 53: 96661.59 + rank 54: 96659.58 + rank 55: 96661.67 + rank 56: 96656.14 + rank 57: 96656.99 + rank 58: 96655.35 + rank 59: 96655.73 + rank 60: 96655.91 + rank 61: 96655.99 + rank 62: 96655.72 + rank 63: 96655.84 + train/valid/test-data-iterators-setup: + rank 0: 23514.95 + rank 1: 26882.88 + rank 2: 26882.68 + rank 3: 26882.72 + rank 4: 26882.76 + rank 5: 26882.96 + rank 6: 26829.47 + rank 7: 26814.90 + rank 8: 26882.56 + rank 9: 27292.12 + rank 10: 27292.07 + rank 11: 27292.09 + rank 12: 27292.09 + rank 13: 27292.29 + rank 14: 26882.54 + rank 15: 26882.61 + rank 16: 27289.20 + rank 17: 27289.04 + rank 18: 27289.17 + rank 19: 27289.03 + rank 20: 27288.96 + rank 21: 27289.07 + rank 22: 27289.07 + rank 23: 27289.07 + rank 24: 28069.29 + rank 25: 28238.46 + rank 26: 28239.20 + rank 27: 28238.45 + rank 28: 28238.50 + rank 29: 28238.31 + rank 30: 28069.50 + rank 31: 28106.84 + rank 32: 28238.07 + rank 33: 28237.98 + rank 34: 28238.10 + rank 35: 28238.11 + rank 36: 28238.08 + rank 37: 28237.90 + rank 38: 28238.00 + rank 39: 28238.07 + rank 40: 28238.50 + rank 41: 28238.54 + rank 42: 28238.33 + rank 43: 28238.39 + rank 44: 28238.66 + rank 45: 28238.56 + rank 46: 28238.29 + rank 47: 28238.29 + rank 48: 28237.91 + rank 49: 28237.91 + rank 50: 28238.23 + rank 51: 28238.06 + rank 52: 28238.02 + rank 53: 28238.06 + rank 54: 28237.93 + rank 55: 28238.03 + rank 56: 28238.12 + rank 57: 28238.18 + rank 58: 28238.11 + rank 59: 28238.13 + rank 60: 28238.12 + rank 61: 28238.02 + rank 62: 28238.18 + rank 63: 28238.08 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + + + +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 + [2026-10-11 02:56:00] iteration 1/ 2384 | consumed samples: 32 | elapsed time per iteration (ms): 404969.6 | throughput per GPU (TFLOP/s/GPU): 12.5 | MFU 4.00% | FLOPs estimator: workload-aware | logical tokens/s: 41428.2 | valid tokens/s: 41428.2 | physical tokens/s: 41624.9 | loss tokens/s: 31194.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.871515E-01 | lm-global-supervised-tokens: 1.263289E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.345376E+01 | lm-token-buffer-mib: 1.167862E+02 | global_load_balancing_loss: 1.003968E+00 | loss scale: 1.0 | grad norm: 0.323 | params norm: 11236.196 | params rms: 0.060840 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11 days, 4:04:02.454443 | finish at 2026-10-22 07:00:02 +times across ranks (ms): + forward-backward: + rank 0: 379184.28 + rank 1: 379122.19 + rank 2: 379122.97 + rank 3: 379184.56 + rank 4: 379102.22 + rank 5: 379193.03 + rank 6: 379122.28 + rank 7: 379189.88 + rank 8: 371255.84 + rank 9: 371241.88 + rank 10: 371261.62 + rank 11: 371247.91 + rank 12: 371248.78 + rank 13: 371226.53 + rank 14: 371243.75 + rank 15: 371243.78 + rank 16: 379177.66 + rank 17: 379116.84 + rank 18: 379117.69 + rank 19: 379178.16 + rank 20: 379097.12 + rank 21: 379186.25 + rank 22: 379116.16 + rank 23: 379184.19 + rank 24: 371249.12 + rank 25: 371232.75 + rank 26: 371252.56 + rank 27: 371239.38 + rank 28: 371239.72 + rank 29: 371217.28 + rank 30: 371235.28 + rank 31: 371235.50 + rank 32: 379184.41 + rank 33: 379124.81 + rank 34: 379126.50 + rank 35: 379186.00 + rank 36: 379105.91 + rank 37: 379193.66 + rank 38: 379125.03 + rank 39: 379192.66 + rank 40: 371241.31 + rank 41: 371227.31 + rank 42: 371247.47 + rank 43: 371233.59 + rank 44: 371234.06 + rank 45: 371211.34 + rank 46: 371228.88 + rank 47: 371229.09 + rank 48: 379185.88 + rank 49: 379124.88 + rank 50: 379123.84 + rank 51: 379185.78 + rank 52: 379105.06 + rank 53: 379193.34 + rank 54: 379123.56 + rank 55: 379191.91 + rank 56: 371241.84 + rank 57: 371227.41 + rank 58: 371245.28 + rank 59: 371231.84 + rank 60: 371234.59 + rank 61: 371211.97 + rank 62: 371229.19 + rank 63: 371229.78 + forward-compute: + rank 0: 171413.70 + rank 1: 171357.88 + rank 2: 171365.62 + rank 3: 171425.31 + rank 4: 171345.86 + rank 5: 171437.08 + rank 6: 171378.20 + rank 7: 171453.14 + rank 8: 163439.38 + rank 9: 163424.80 + rank 10: 163453.91 + rank 11: 163461.41 + rank 12: 163459.08 + rank 13: 163454.52 + rank 14: 163430.62 + rank 15: 163432.17 + rank 16: 217602.09 + rank 17: 217543.62 + rank 18: 217543.70 + rank 19: 217605.22 + rank 20: 217524.03 + rank 21: 217611.59 + rank 22: 217542.33 + rank 23: 217610.58 + rank 24: 209809.67 + rank 25: 209730.58 + rank 26: 209750.86 + rank 27: 209735.19 + rank 28: 209737.78 + rank 29: 209714.97 + rank 30: 209732.67 + rank 31: 209730.11 + rank 32: 221132.86 + rank 33: 221077.67 + rank 34: 221080.86 + rank 35: 221146.48 + rank 36: 221066.14 + rank 37: 221157.30 + rank 38: 221082.28 + rank 39: 221150.36 + rank 40: 213183.48 + rank 41: 213170.00 + rank 42: 213186.09 + rank 43: 213173.23 + rank 44: 213175.12 + rank 45: 213155.67 + rank 46: 213169.83 + rank 47: 213165.67 + rank 48: 152733.11 + rank 49: 152664.14 + rank 50: 152668.50 + rank 51: 152730.02 + rank 52: 152707.19 + rank 53: 152771.09 + rank 54: 152603.67 + rank 55: 152772.77 + rank 56: 144703.89 + rank 57: 144690.25 + rank 58: 144761.84 + rank 59: 144690.02 + rank 60: 144701.95 + rank 61: 144682.72 + rank 62: 144680.88 + rank 63: 144689.20 + backward-compute: + rank 0: 207701.58 + rank 1: 207692.88 + rank 2: 207686.62 + rank 3: 207690.12 + rank 4: 207675.91 + rank 5: 207683.84 + rank 6: 207672.92 + rank 7: 207668.38 + rank 8: 207727.86 + rank 9: 207731.53 + rank 10: 207721.64 + rank 11: 207702.75 + rank 12: 207705.83 + rank 13: 207684.34 + rank 14: 207726.12 + rank 15: 207723.44 + rank 16: 161501.39 + rank 17: 161500.16 + rank 18: 161502.73 + rank 19: 161502.06 + rank 20: 161492.23 + rank 21: 161501.98 + rank 22: 161495.34 + rank 23: 161498.17 + rank 24: 161324.81 + rank 25: 161411.09 + rank 26: 161408.86 + rank 27: 161414.41 + rank 28: 161412.27 + rank 29: 161414.59 + rank 30: 161411.30 + rank 31: 161410.41 + rank 32: 157955.86 + rank 33: 157953.09 + rank 34: 157948.70 + rank 35: 157945.84 + rank 36: 157944.58 + rank 37: 157942.70 + rank 38: 157947.25 + rank 39: 157944.69 + rank 40: 157978.36 + rank 41: 157977.08 + rank 42: 157978.61 + rank 43: 157977.70 + rank 44: 157974.52 + rank 45: 157972.23 + rank 46: 157981.50 + rank 47: 157984.38 + rank 48: 226361.30 + rank 49: 226381.03 + rank 50: 226377.86 + rank 51: 226377.92 + rank 52: 226309.64 + rank 53: 226340.36 + rank 54: 226442.80 + rank 55: 226333.27 + rank 56: 226483.66 + rank 57: 226480.12 + rank 58: 226431.16 + rank 59: 226488.52 + rank 60: 226473.62 + rank 61: 226472.30 + rank 62: 226494.12 + rank 63: 226487.20 + batch-generator: + rank 0: 44.99 + rank 1: 38.18 + rank 2: 46.12 + rank 3: 45.80 + rank 4: 40.63 + rank 5: 53.85 + rank 6: 38.99 + rank 7: 50.38 + rank 8: 44.98 + rank 9: 46.08 + rank 10: 49.29 + rank 11: 45.13 + rank 12: 56.44 + rank 13: 47.22 + rank 14: 42.54 + rank 15: 51.83 + rank 16: 38.67 + rank 17: 36.15 + rank 18: 38.70 + rank 19: 36.58 + rank 20: 41.24 + rank 21: 41.78 + rank 22: 37.18 + rank 23: 39.82 + rank 24: 74.22 + rank 25: 52.95 + rank 26: 73.50 + rank 27: 60.53 + rank 28: 63.88 + rank 29: 48.55 + rank 30: 55.78 + rank 31: 56.25 + rank 32: 41.64 + rank 33: 40.18 + rank 34: 44.82 + rank 35: 45.40 + rank 36: 41.24 + rank 37: 43.43 + rank 38: 43.74 + rank 39: 41.83 + rank 40: 45.70 + rank 41: 37.32 + rank 42: 50.95 + rank 43: 45.77 + rank 44: 40.51 + rank 45: 36.87 + rank 46: 37.89 + rank 47: 45.70 + rank 48: 49.13 + rank 49: 42.03 + rank 50: 38.40 + rank 51: 43.17 + rank 52: 44.61 + rank 53: 48.03 + rank 54: 39.32 + rank 55: 48.76 + rank 56: 42.92 + rank 57: 42.74 + rank 58: 43.80 + rank 59: 40.48 + rank 60: 41.22 + rank 61: 44.83 + rank 62: 40.95 + rank 63: 42.03 + embedding-grads-all-reduce: + rank 0: 0.02 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.02 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.02 + rank 13: 0.03 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.05 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.04 + rank 36: 0.04 + rank 37: 0.03 + rank 38: 0.04 + rank 39: 0.03 + rank 40: 0.09 + rank 41: 0.08 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.04 + rank 47: 0.04 + rank 48: 0.04 + rank 49: 0.04 + rank 50: 0.03 + rank 51: 0.04 + rank 52: 0.04 + rank 53: 0.03 + rank 54: 0.04 + rank 55: 0.06 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.84 + rank 1: 0.81 + rank 2: 0.96 + rank 3: 0.86 + rank 4: 1.65 + rank 5: 0.87 + rank 6: 0.85 + rank 7: 1.08 + rank 8: 1.01 + rank 9: 0.84 + rank 10: 1.21 + rank 11: 1.24 + rank 12: 0.95 + rank 13: 0.86 + rank 14: 0.91 + rank 15: 0.85 + rank 16: 1.07 + rank 17: 0.82 + rank 18: 0.89 + rank 19: 0.83 + rank 20: 0.97 + rank 21: 1.12 + rank 22: 1.03 + rank 23: 0.90 + rank 24: 1.67 + rank 25: 1.14 + rank 26: 1.65 + rank 27: 1.38 + rank 28: 1.29 + rank 29: 1.28 + rank 30: 1.39 + rank 31: 1.20 + rank 32: 1.14 + rank 33: 1.09 + rank 34: 1.19 + rank 35: 1.22 + rank 36: 1.18 + rank 37: 1.18 + rank 38: 1.17 + rank 39: 1.49 + rank 40: 2.24 + rank 41: 1.66 + rank 42: 1.01 + rank 43: 0.85 + rank 44: 1.03 + rank 45: 0.82 + rank 46: 1.69 + rank 47: 1.74 + rank 48: 1.55 + rank 49: 1.39 + rank 50: 0.87 + rank 51: 1.23 + rank 52: 1.44 + rank 53: 1.47 + rank 54: 1.03 + rank 55: 2.89 + rank 56: 0.85 + rank 57: 0.83 + rank 58: 0.78 + rank 59: 0.78 + rank 60: 0.83 + rank 61: 0.78 + rank 62: 0.85 + rank 63: 0.80 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.03 + rank 20: 0.07 + rank 21: 0.02 + rank 22: 0.03 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.02 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.03 + optimizer-copy-to-main-grad: + rank 0: 2.48 + rank 1: 2.21 + rank 2: 2.14 + rank 3: 2.49 + rank 4: 2.31 + rank 5: 3.35 + rank 6: 2.29 + rank 7: 2.39 + rank 8: 2.21 + rank 9: 2.58 + rank 10: 2.17 + rank 11: 2.17 + rank 12: 2.21 + rank 13: 2.20 + rank 14: 2.30 + rank 15: 2.40 + rank 16: 3.00 + rank 17: 2.76 + rank 18: 2.43 + rank 19: 2.80 + rank 20: 2.95 + rank 21: 2.84 + rank 22: 3.18 + rank 23: 2.81 + rank 24: 2.68 + rank 25: 2.55 + rank 26: 2.04 + rank 27: 2.55 + rank 28: 2.67 + rank 29: 2.65 + rank 30: 2.64 + rank 31: 2.12 + rank 32: 2.45 + rank 33: 2.45 + rank 34: 2.46 + rank 35: 2.75 + rank 36: 2.69 + rank 37: 2.58 + rank 38: 2.79 + rank 39: 2.04 + rank 40: 2.10 + rank 41: 2.30 + rank 42: 2.21 + rank 43: 2.31 + rank 44: 2.30 + rank 45: 2.47 + rank 46: 2.08 + rank 47: 2.06 + rank 48: 3.18 + rank 49: 2.32 + rank 50: 3.01 + rank 51: 2.38 + rank 52: 2.63 + rank 53: 2.51 + rank 54: 2.23 + rank 55: 3.14 + rank 56: 2.22 + rank 57: 2.90 + rank 58: 2.31 + rank 59: 2.19 + rank 60: 2.21 + rank 61: 2.18 + rank 62: 2.30 + rank 63: 2.24 + optimizer-inner-step: + rank 0: 105.13 + rank 1: 99.71 + rank 2: 99.78 + rank 3: 100.24 + rank 4: 99.86 + rank 5: 99.69 + rank 6: 99.68 + rank 7: 99.74 + rank 8: 107.79 + rank 9: 107.42 + rank 10: 107.77 + rank 11: 107.74 + rank 12: 107.61 + rank 13: 107.79 + rank 14: 107.69 + rank 15: 107.52 + rank 16: 103.45 + rank 17: 103.68 + rank 18: 103.61 + rank 19: 103.58 + rank 20: 104.53 + rank 21: 103.58 + rank 22: 103.37 + rank 23: 103.51 + rank 24: 120.15 + rank 25: 119.74 + rank 26: 120.12 + rank 27: 119.79 + rank 28: 119.82 + rank 29: 119.74 + rank 30: 119.77 + rank 31: 119.98 + rank 32: 103.00 + rank 33: 103.06 + rank 34: 103.00 + rank 35: 102.70 + rank 36: 103.01 + rank 37: 103.02 + rank 38: 102.90 + rank 39: 103.11 + rank 40: 99.27 + rank 41: 99.15 + rank 42: 99.16 + rank 43: 99.13 + rank 44: 99.25 + rank 45: 99.14 + rank 46: 99.30 + rank 47: 99.22 + rank 48: 111.62 + rank 49: 111.53 + rank 50: 111.06 + rank 51: 111.56 + rank 52: 111.49 + rank 53: 111.48 + rank 54: 111.52 + rank 55: 111.73 + rank 56: 98.15 + rank 57: 97.83 + rank 58: 98.11 + rank 59: 98.23 + rank 60: 98.15 + rank 61: 98.23 + rank 62: 98.03 + rank 63: 98.23 + optimizer-copy-main-to-model-params: + rank 0: 4.66 + rank 1: 4.16 + rank 2: 3.99 + rank 3: 5.97 + rank 4: 4.94 + rank 5: 4.32 + rank 6: 4.17 + rank 7: 4.37 + rank 8: 4.12 + rank 9: 4.97 + rank 10: 4.07 + rank 11: 4.32 + rank 12: 4.20 + rank 13: 4.30 + rank 14: 4.29 + rank 15: 4.76 + rank 16: 4.83 + rank 17: 4.47 + rank 18: 4.42 + rank 19: 4.54 + rank 20: 10.99 + rank 21: 4.66 + rank 22: 5.12 + rank 23: 4.70 + rank 24: 4.75 + rank 25: 4.75 + rank 26: 4.38 + rank 27: 4.65 + rank 28: 4.82 + rank 29: 4.87 + rank 30: 4.83 + rank 31: 4.66 + rank 32: 4.88 + rank 33: 4.57 + rank 34: 4.70 + rank 35: 5.22 + rank 36: 5.08 + rank 37: 4.90 + rank 38: 5.32 + rank 39: 4.57 + rank 40: 3.96 + rank 41: 4.56 + rank 42: 4.21 + rank 43: 4.59 + rank 44: 4.20 + rank 45: 4.92 + rank 46: 3.97 + rank 47: 4.30 + rank 48: 4.83 + rank 49: 4.51 + rank 50: 5.47 + rank 51: 4.50 + rank 52: 4.73 + rank 53: 4.42 + rank 54: 4.28 + rank 55: 4.98 + rank 56: 4.07 + rank 57: 5.14 + rank 58: 4.20 + rank 59: 4.11 + rank 60: 4.04 + rank 61: 4.12 + rank 62: 4.19 + rank 63: 4.11 + optimizer: + rank 0: 129.04 + rank 1: 122.92 + rank 2: 122.78 + rank 3: 125.53 + rank 4: 123.77 + rank 5: 123.04 + rank 6: 122.86 + rank 7: 123.16 + rank 8: 130.97 + rank 9: 131.68 + rank 10: 130.85 + rank 11: 131.11 + rank 12: 130.89 + rank 13: 131.17 + rank 14: 131.11 + rank 15: 131.51 + rank 16: 127.82 + rank 17: 127.49 + rank 18: 127.39 + rank 19: 127.52 + rank 20: 135.19 + rank 21: 127.65 + rank 22: 128.09 + rank 23: 127.62 + rank 24: 143.97 + rank 25: 143.63 + rank 26: 143.55 + rank 27: 143.59 + rank 28: 143.86 + rank 29: 143.80 + rank 30: 143.80 + rank 31: 143.68 + rank 32: 126.88 + rank 33: 126.56 + rank 34: 126.65 + rank 35: 127.07 + rank 36: 127.17 + rank 37: 126.92 + rank 38: 127.36 + rank 39: 126.61 + rank 40: 122.18 + rank 41: 122.73 + rank 42: 122.39 + rank 43: 122.75 + rank 44: 122.47 + rank 45: 123.19 + rank 46: 122.18 + rank 47: 122.45 + rank 48: 135.48 + rank 49: 135.02 + rank 50: 135.95 + rank 51: 135.08 + rank 52: 135.29 + rank 53: 134.86 + rank 54: 134.79 + rank 55: 135.73 + rank 56: 121.17 + rank 57: 122.28 + rank 58: 121.32 + rank 59: 121.29 + rank 60: 121.15 + rank 61: 121.29 + rank 62: 121.23 + rank 63: 121.25 + interval-time: + rank 0: 2.95 + rank 1: 2.97 + rank 2: 2.90 + rank 3: 2.94 + rank 4: 2.99 + rank 5: 2.95 + rank 6: 2.93 + rank 7: 2.94 + rank 8: 1.86 + rank 9: 1.86 + rank 10: 1.87 + rank 11: 1.86 + rank 12: 1.96 + rank 13: 1.86 + rank 14: 1.88 + rank 15: 1.87 + rank 16: 1.76 + rank 17: 1.78 + rank 18: 1.78 + rank 19: 1.77 + rank 20: 1.83 + rank 21: 1.76 + rank 22: 1.76 + rank 23: 1.79 + rank 24: 2.48 + rank 25: 1.88 + rank 26: 2.29 + rank 27: 2.30 + rank 28: 2.58 + rank 29: 1.82 + rank 30: 1.89 + rank 31: 1.94 + rank 32: 2.21 + rank 33: 1.91 + rank 34: 2.07 + rank 35: 1.86 + rank 36: 2.18 + rank 37: 1.95 + rank 38: 2.20 + rank 39: 1.98 + rank 40: 1.92 + rank 41: 1.92 + rank 42: 1.80 + rank 43: 1.89 + rank 44: 1.92 + rank 45: 1.96 + rank 46: 1.93 + rank 47: 1.87 + rank 48: 2.35 + rank 49: 1.96 + rank 50: 1.94 + rank 51: 1.92 + rank 52: 1.93 + rank 53: 1.91 + rank 54: 2.01 + rank 55: 2.33 + rank 56: 1.70 + rank 57: 1.67 + rank 58: 1.86 + rank 59: 1.83 + rank 60: 1.81 + rank 61: 1.91 + rank 62: 1.66 + rank 63: 3.17 + [2026-10-11 02:59:00] iteration 2/ 2384 | consumed samples: 64 | elapsed time per iteration (ms): 180724.0 | throughput per GPU (TFLOP/s/GPU): 27.4 | MFU 8.77% | FLOPs estimator: workload-aware | logical tokens/s: 92833.2 | valid tokens/s: 92833.2 | physical tokens/s: 93192.3 | loss tokens/s: 69868.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.678903E-01 | lm-global-supervised-tokens: 1.262682E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.886211E-02 | lm-token-buffer-mib: 1.162751E+02 | global_load_balancing_loss: 9.996409E-01 | loss scale: 1.0 | grad norm: 0.309 | params norm: 11236.140 | params rms: 0.060840 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:34:44.659947 | finish at 2026-10-16 02:33:45 +times across ranks (ms): + forward-backward: + rank 0: 180491.55 + rank 1: 180491.81 + rank 2: 180492.62 + rank 3: 180500.59 + rank 4: 180493.84 + rank 5: 180493.75 + rank 6: 180470.00 + rank 7: 180491.56 + rank 8: 180487.97 + rank 9: 180488.75 + rank 10: 180489.61 + rank 11: 180489.73 + rank 12: 180490.22 + rank 13: 180490.72 + rank 14: 180487.91 + rank 15: 180486.84 + rank 16: 180487.19 + rank 17: 180487.88 + rank 18: 180487.84 + rank 19: 180488.50 + rank 20: 180488.09 + rank 21: 180490.23 + rank 22: 180465.12 + rank 23: 180487.41 + rank 24: 180480.44 + rank 25: 180482.02 + rank 26: 180483.44 + rank 27: 180482.20 + rank 28: 180485.50 + rank 29: 180483.91 + rank 30: 180480.45 + rank 31: 180478.83 + rank 32: 180478.98 + rank 33: 180477.02 + rank 34: 180478.03 + rank 35: 180478.16 + rank 36: 180478.27 + rank 37: 180479.91 + rank 38: 180456.19 + rank 39: 180478.31 + rank 40: 180479.38 + rank 41: 180476.72 + rank 42: 180477.66 + rank 43: 180475.64 + rank 44: 180476.41 + rank 45: 180474.89 + rank 46: 180481.19 + rank 47: 180478.06 + rank 48: 180499.39 + rank 49: 180497.09 + rank 50: 180496.81 + rank 51: 180497.12 + rank 52: 180495.47 + rank 53: 180496.78 + rank 54: 180477.83 + rank 55: 180501.16 + rank 56: 180494.44 + rank 57: 180494.70 + rank 58: 180494.98 + rank 59: 180493.75 + rank 60: 180494.66 + rank 61: 180492.78 + rank 62: 180496.08 + rank 63: 180494.09 + forward-compute: + rank 0: 69073.80 + rank 1: 69065.13 + rank 2: 69083.31 + rank 3: 69077.10 + rank 4: 69075.86 + rank 5: 69073.77 + rank 6: 69052.81 + rank 7: 69094.38 + rank 8: 69087.29 + rank 9: 69080.24 + rank 10: 69090.05 + rank 11: 69100.61 + rank 12: 69099.61 + rank 13: 69101.77 + rank 14: 69093.77 + rank 15: 69090.89 + rank 16: 68581.02 + rank 17: 68581.00 + rank 18: 68575.89 + rank 19: 68586.43 + rank 20: 68575.02 + rank 21: 68581.04 + rank 22: 68560.06 + rank 23: 68583.88 + rank 24: 68573.40 + rank 25: 68570.48 + rank 26: 68573.54 + rank 27: 68570.25 + rank 28: 68579.61 + rank 29: 68581.65 + rank 30: 68575.76 + rank 31: 68571.21 + rank 32: 63491.96 + rank 33: 63500.49 + rank 34: 63510.18 + rank 35: 63516.66 + rank 36: 63520.63 + rank 37: 63523.18 + rank 38: 63494.26 + rank 39: 63503.76 + rank 40: 63512.81 + rank 41: 63500.88 + rank 42: 63501.30 + rank 43: 63510.84 + rank 44: 63513.92 + rank 45: 63518.51 + rank 46: 63520.23 + rank 47: 63519.76 + rank 48: 50973.52 + rank 49: 50967.90 + rank 50: 50972.53 + rank 51: 50966.80 + rank 52: 50993.00 + rank 53: 50986.21 + rank 54: 50907.62 + rank 55: 50962.84 + rank 56: 50946.27 + rank 57: 50943.08 + rank 58: 50942.74 + rank 59: 50926.29 + rank 60: 50947.35 + rank 61: 50960.10 + rank 62: 50947.09 + rank 63: 50963.16 + backward-compute: + rank 0: 111393.12 + rank 1: 111396.70 + rank 2: 111381.50 + rank 3: 111389.46 + rank 4: 111376.18 + rank 5: 111394.22 + rank 6: 111391.25 + rank 7: 111371.65 + rank 8: 111374.38 + rank 9: 111383.35 + rank 10: 111363.68 + rank 11: 111363.96 + rank 12: 111364.05 + rank 13: 111359.78 + rank 14: 111368.52 + rank 15: 111370.59 + rank 16: 111873.90 + rank 17: 111881.56 + rank 18: 111887.08 + rank 19: 111875.92 + rank 20: 111885.88 + rank 21: 111884.46 + rank 22: 111879.54 + rank 23: 111878.52 + rank 24: 111867.06 + rank 25: 111882.86 + rank 26: 111877.63 + rank 27: 111878.95 + rank 28: 111872.92 + rank 29: 111872.33 + rank 30: 111873.83 + rank 31: 111877.34 + rank 32: 116955.77 + rank 33: 116946.75 + rank 34: 116939.17 + rank 35: 116931.76 + rank 36: 116927.41 + rank 37: 116927.02 + rank 38: 116932.83 + rank 39: 116946.14 + rank 40: 116942.30 + rank 41: 116952.91 + rank 42: 116952.36 + rank 43: 116941.97 + rank 44: 116939.33 + rank 45: 116932.98 + rank 46: 116937.87 + rank 47: 116934.82 + rank 48: 129495.67 + rank 49: 129502.83 + rank 50: 129498.31 + rank 51: 129504.51 + rank 52: 129472.51 + rank 53: 129483.79 + rank 54: 129539.92 + rank 55: 129504.34 + rank 56: 129525.17 + rank 57: 129527.81 + rank 58: 129528.51 + rank 59: 129543.97 + rank 60: 129523.98 + rank 61: 129505.90 + rank 62: 129525.73 + rank 63: 129507.39 + batch-generator: + rank 0: 15.81 + rank 1: 15.69 + rank 2: 15.39 + rank 3: 15.95 + rank 4: 16.22 + rank 5: 14.94 + rank 6: 15.50 + rank 7: 15.58 + rank 8: 15.66 + rank 9: 15.40 + rank 10: 21.25 + rank 11: 19.56 + rank 12: 17.50 + rank 13: 18.67 + rank 14: 17.14 + rank 15: 15.84 + rank 16: 16.52 + rank 17: 16.24 + rank 18: 14.19 + rank 19: 16.01 + rank 20: 15.45 + rank 21: 13.85 + rank 22: 15.85 + rank 23: 14.96 + rank 24: 22.21 + rank 25: 20.20 + rank 26: 22.55 + rank 27: 23.01 + rank 28: 19.30 + rank 29: 17.17 + rank 30: 18.67 + rank 31: 18.99 + rank 32: 21.70 + rank 33: 22.33 + rank 34: 21.94 + rank 35: 22.39 + rank 36: 20.49 + rank 37: 19.87 + rank 38: 20.94 + rank 39: 19.54 + rank 40: 14.42 + rank 41: 14.42 + rank 42: 15.45 + rank 43: 15.48 + rank 44: 13.80 + rank 45: 13.96 + rank 46: 14.20 + rank 47: 14.20 + rank 48: 21.52 + rank 49: 19.02 + rank 50: 18.23 + rank 51: 15.46 + rank 52: 16.92 + rank 53: 19.07 + rank 54: 15.59 + rank 55: 21.93 + rank 56: 15.15 + rank 57: 15.30 + rank 58: 14.36 + rank 59: 14.75 + rank 60: 15.34 + rank 61: 16.34 + rank 62: 14.42 + rank 63: 14.77 + embedding-grads-all-reduce: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.03 + rank 4: 0.04 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.03 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.04 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.02 + all-grads-sync: + rank 0: 1.02 + rank 1: 0.81 + rank 2: 0.80 + rank 3: 1.21 + rank 4: 1.16 + rank 5: 1.09 + rank 6: 0.83 + rank 7: 0.80 + rank 8: 1.19 + rank 9: 0.77 + rank 10: 1.18 + rank 11: 0.83 + rank 12: 0.83 + rank 13: 0.81 + rank 14: 0.83 + rank 15: 0.75 + rank 16: 0.74 + rank 17: 1.02 + rank 18: 0.81 + rank 19: 0.86 + rank 20: 0.76 + rank 21: 0.82 + rank 22: 0.77 + rank 23: 1.04 + rank 24: 1.42 + rank 25: 1.43 + rank 26: 1.32 + rank 27: 1.43 + rank 28: 1.13 + rank 29: 1.43 + rank 30: 1.15 + rank 31: 1.33 + rank 32: 2.02 + rank 33: 1.53 + rank 34: 1.57 + rank 35: 1.77 + rank 36: 1.65 + rank 37: 2.09 + rank 38: 1.27 + rank 39: 1.58 + rank 40: 1.28 + rank 41: 0.85 + rank 42: 0.82 + rank 43: 0.78 + rank 44: 1.09 + rank 45: 0.83 + rank 46: 1.01 + rank 47: 0.84 + rank 48: 1.38 + rank 49: 0.97 + rank 50: 1.00 + rank 51: 1.10 + rank 52: 1.11 + rank 53: 0.97 + rank 54: 1.21 + rank 55: 2.11 + rank 56: 0.77 + rank 57: 0.78 + rank 58: 0.79 + rank 59: 0.78 + rank 60: 0.84 + rank 61: 0.76 + rank 62: 0.78 + rank 63: 0.83 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.05 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.02 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.03 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.13 + rank 1: 1.76 + rank 2: 1.77 + rank 3: 3.16 + rank 4: 1.94 + rank 5: 1.87 + rank 6: 1.84 + rank 7: 2.00 + rank 8: 1.95 + rank 9: 2.18 + rank 10: 1.82 + rank 11: 1.82 + rank 12: 1.88 + rank 13: 1.76 + rank 14: 1.95 + rank 15: 2.04 + rank 16: 2.33 + rank 17: 2.14 + rank 18: 2.15 + rank 19: 2.20 + rank 20: 2.25 + rank 21: 2.26 + rank 22: 2.53 + rank 23: 2.27 + rank 24: 2.98 + rank 25: 2.29 + rank 26: 2.26 + rank 27: 2.22 + rank 28: 2.39 + rank 29: 2.39 + rank 30: 2.36 + rank 31: 2.12 + rank 32: 2.05 + rank 33: 1.83 + rank 34: 2.10 + rank 35: 2.33 + rank 36: 2.17 + rank 37: 2.06 + rank 38: 2.28 + rank 39: 1.96 + rank 40: 1.78 + rank 41: 1.82 + rank 42: 1.90 + rank 43: 1.96 + rank 44: 1.89 + rank 45: 2.08 + rank 46: 1.83 + rank 47: 1.73 + rank 48: 2.31 + rank 49: 1.86 + rank 50: 2.50 + rank 51: 1.98 + rank 52: 2.29 + rank 53: 1.90 + rank 54: 1.86 + rank 55: 2.04 + rank 56: 1.88 + rank 57: 2.54 + rank 58: 1.89 + rank 59: 1.91 + rank 60: 1.79 + rank 61: 1.77 + rank 62: 1.93 + rank 63: 1.80 + optimizer-inner-step: + rank 0: 42.28 + rank 1: 42.04 + rank 2: 41.73 + rank 3: 46.33 + rank 4: 41.76 + rank 5: 41.84 + rank 6: 42.30 + rank 7: 42.21 + rank 8: 41.94 + rank 9: 42.02 + rank 10: 41.89 + rank 11: 41.98 + rank 12: 41.91 + rank 13: 42.18 + rank 14: 41.95 + rank 15: 41.90 + rank 16: 42.42 + rank 17: 42.07 + rank 18: 42.18 + rank 19: 42.00 + rank 20: 42.08 + rank 21: 41.93 + rank 22: 42.45 + rank 23: 42.18 + rank 24: 41.77 + rank 25: 42.01 + rank 26: 41.95 + rank 27: 42.09 + rank 28: 46.22 + rank 29: 41.86 + rank 30: 41.85 + rank 31: 42.07 + rank 32: 41.98 + rank 33: 41.80 + rank 34: 41.90 + rank 35: 47.95 + rank 36: 42.40 + rank 37: 42.01 + rank 38: 41.85 + rank 39: 41.85 + rank 40: 41.94 + rank 41: 41.99 + rank 42: 42.12 + rank 43: 42.15 + rank 44: 41.70 + rank 45: 44.35 + rank 46: 42.09 + rank 47: 41.60 + rank 48: 44.27 + rank 49: 41.91 + rank 50: 53.50 + rank 51: 41.66 + rank 52: 41.91 + rank 53: 41.91 + rank 54: 41.98 + rank 55: 41.89 + rank 56: 41.60 + rank 57: 47.59 + rank 58: 41.93 + rank 59: 41.79 + rank 60: 42.10 + rank 61: 41.69 + rank 62: 41.81 + rank 63: 41.79 + optimizer-copy-main-to-model-params: + rank 0: 4.45 + rank 1: 3.89 + rank 2: 3.89 + rank 3: 4.50 + rank 4: 3.83 + rank 5: 3.95 + rank 6: 4.05 + rank 7: 4.11 + rank 8: 3.95 + rank 9: 4.49 + rank 10: 3.92 + rank 11: 3.87 + rank 12: 4.01 + rank 13: 3.84 + rank 14: 4.04 + rank 15: 4.33 + rank 16: 4.47 + rank 17: 4.09 + rank 18: 4.16 + rank 19: 4.06 + rank 20: 4.14 + rank 21: 4.26 + rank 22: 4.73 + rank 23: 4.26 + rank 24: 4.10 + rank 25: 4.40 + rank 26: 4.07 + rank 27: 4.20 + rank 28: 4.55 + rank 29: 4.44 + rank 30: 4.46 + rank 31: 4.18 + rank 32: 4.21 + rank 33: 3.98 + rank 34: 4.27 + rank 35: 4.63 + rank 36: 4.42 + rank 37: 4.23 + rank 38: 4.68 + rank 39: 4.02 + rank 40: 3.83 + rank 41: 3.99 + rank 42: 4.05 + rank 43: 4.08 + rank 44: 4.01 + rank 45: 4.39 + rank 46: 3.88 + rank 47: 3.76 + rank 48: 4.78 + rank 49: 4.07 + rank 50: 5.06 + rank 51: 4.18 + rank 52: 4.48 + rank 53: 4.08 + rank 54: 3.96 + rank 55: 4.06 + rank 56: 3.84 + rank 57: 4.92 + rank 58: 3.97 + rank 59: 3.80 + rank 60: 3.81 + rank 61: 3.75 + rank 62: 3.98 + rank 63: 3.73 + optimizer: + rank 0: 55.05 + rank 1: 54.07 + rank 2: 53.70 + rank 3: 59.64 + rank 4: 53.71 + rank 5: 53.91 + rank 6: 54.51 + rank 7: 54.49 + rank 8: 54.06 + rank 9: 54.83 + rank 10: 53.90 + rank 11: 53.96 + rank 12: 54.04 + rank 13: 54.15 + rank 14: 54.18 + rank 15: 54.48 + rank 16: 55.41 + rank 17: 54.47 + rank 18: 54.71 + rank 19: 54.42 + rank 20: 54.57 + rank 21: 54.66 + rank 22: 55.78 + rank 23: 54.85 + rank 24: 53.95 + rank 25: 54.61 + rank 26: 54.12 + rank 27: 54.45 + rank 28: 59.05 + rank 29: 54.55 + rank 30: 54.55 + rank 31: 54.40 + rank 32: 54.30 + rank 33: 53.82 + rank 34: 54.27 + rank 35: 60.87 + rank 36: 55.62 + rank 37: 54.36 + rank 38: 54.79 + rank 39: 53.91 + rank 40: 53.83 + rank 41: 54.13 + rank 42: 54.31 + rank 43: 54.40 + rank 44: 53.85 + rank 45: 57.03 + rank 46: 54.01 + rank 47: 53.43 + rank 48: 57.26 + rank 49: 54.06 + rank 50: 67.11 + rank 51: 53.95 + rank 52: 54.61 + rank 53: 54.06 + rank 54: 54.03 + rank 55: 54.06 + rank 56: 53.52 + rank 57: 61.22 + rank 58: 54.01 + rank 59: 53.68 + rank 60: 53.98 + rank 61: 53.51 + rank 62: 53.94 + rank 63: 53.58 + interval-time: + rank 0: 0.68 + rank 1: 0.69 + rank 2: 0.80 + rank 3: 0.70 + rank 4: 0.86 + rank 5: 0.69 + rank 6: 0.76 + rank 7: 0.79 + rank 8: 0.81 + rank 9: 0.70 + rank 10: 0.81 + rank 11: 0.71 + rank 12: 0.69 + rank 13: 0.77 + rank 14: 0.82 + rank 15: 0.69 + rank 16: 0.78 + rank 17: 0.78 + rank 18: 0.76 + rank 19: 0.79 + rank 20: 0.80 + rank 21: 0.71 + rank 22: 0.69 + rank 23: 0.77 + rank 24: 1.04 + rank 25: 0.79 + rank 26: 0.83 + rank 27: 0.80 + rank 28: 0.91 + rank 29: 0.76 + rank 30: 0.78 + rank 31: 0.82 + rank 32: 0.78 + rank 33: 0.77 + rank 34: 0.79 + rank 35: 0.77 + rank 36: 0.91 + rank 37: 0.81 + rank 38: 0.90 + rank 39: 0.80 + rank 40: 0.71 + rank 41: 0.69 + rank 42: 0.80 + rank 43: 0.72 + rank 44: 0.71 + rank 45: 0.79 + rank 46: 0.70 + rank 47: 0.69 + rank 48: 0.88 + rank 49: 0.79 + rank 50: 0.78 + rank 51: 0.78 + rank 52: 0.83 + rank 53: 0.76 + rank 54: 0.78 + rank 55: 0.88 + rank 56: 0.71 + rank 57: 0.73 + rank 58: 0.70 + rank 59: 0.70 + rank 60: 0.69 + rank 61: 0.72 + rank 62: 0.80 + rank 63: 2.22 + [2026-10-11 03:02:06] iteration 3/ 2384 | consumed samples: 96 | elapsed time per iteration (ms): 185727.1 | throughput per GPU (TFLOP/s/GPU): 31.8 | MFU 10.18% | FLOPs estimator: workload-aware | logical tokens/s: 90332.6 | valid tokens/s: 90332.6 | physical tokens/s: 90744.3 | loss tokens/s: 63457.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.465191E-01 | lm-global-supervised-tokens: 1.178575E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.666467E-02 | lm-token-buffer-mib: 1.164760E+02 | global_load_balancing_loss: 1.002224E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11236.084 | params rms: 0.060840 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:50:16.120967 | finish at 2026-10-16 05:52:22 +times across ranks (ms): + forward-backward: + rank 0: 185507.52 + rank 1: 185510.17 + rank 2: 185510.45 + rank 3: 185510.25 + rank 4: 185512.53 + rank 5: 185511.22 + rank 6: 185507.73 + rank 7: 185508.78 + rank 8: 185503.27 + rank 9: 185506.17 + rank 10: 185506.12 + rank 11: 185506.17 + rank 12: 185507.22 + rank 13: 185506.58 + rank 14: 185505.02 + rank 15: 185504.88 + rank 16: 185501.44 + rank 17: 185504.19 + rank 18: 185504.03 + rank 19: 185504.72 + rank 20: 185505.20 + rank 21: 185506.48 + rank 22: 185502.22 + rank 23: 185502.91 + rank 24: 185494.66 + rank 25: 185498.05 + rank 26: 185498.58 + rank 27: 185496.91 + rank 28: 185498.73 + rank 29: 185498.72 + rank 30: 185495.31 + rank 31: 185494.62 + rank 32: 185492.83 + rank 33: 185495.48 + rank 34: 185495.00 + rank 35: 185494.45 + rank 36: 185496.02 + rank 37: 185496.19 + rank 38: 185496.95 + rank 39: 185496.36 + rank 40: 185496.11 + rank 41: 185497.33 + rank 42: 185496.36 + rank 43: 185494.73 + rank 44: 185495.14 + rank 45: 185493.03 + rank 46: 185498.30 + rank 47: 185497.52 + rank 48: 185512.50 + rank 49: 185512.95 + rank 50: 185512.08 + rank 51: 185511.30 + rank 52: 185510.05 + rank 53: 185511.42 + rank 54: 185513.31 + rank 55: 185514.62 + rank 56: 185510.11 + rank 57: 185510.69 + rank 58: 185510.31 + rank 59: 185509.28 + rank 60: 185508.70 + rank 61: 185508.12 + rank 62: 185512.56 + rank 63: 185511.81 + forward-compute: + rank 0: 66882.82 + rank 1: 66897.22 + rank 2: 66909.44 + rank 3: 66903.80 + rank 4: 66902.12 + rank 5: 66894.75 + rank 6: 66908.31 + rank 7: 66980.94 + rank 8: 66927.47 + rank 9: 66898.22 + rank 10: 66896.99 + rank 11: 66904.10 + rank 12: 66921.88 + rank 13: 66935.77 + rank 14: 66902.75 + rank 15: 66901.54 + rank 16: 110573.06 + rank 17: 110580.90 + rank 18: 110581.27 + rank 19: 110588.38 + rank 20: 110586.89 + rank 21: 110584.66 + rank 22: 110577.89 + rank 23: 110589.68 + rank 24: 110576.95 + rank 25: 110577.30 + rank 26: 110581.52 + rank 27: 110577.23 + rank 28: 110588.74 + rank 29: 110586.36 + rank 30: 110583.11 + rank 31: 110582.98 + rank 32: 96592.89 + rank 33: 96558.82 + rank 34: 96559.25 + rank 35: 96554.28 + rank 36: 96558.97 + rank 37: 96551.82 + rank 38: 96559.52 + rank 39: 96557.21 + rank 40: 96558.84 + rank 41: 96552.63 + rank 42: 96552.88 + rank 43: 96553.31 + rank 44: 96549.09 + rank 45: 96550.71 + rank 46: 96553.63 + rank 47: 96550.03 + rank 48: 78759.52 + rank 49: 78756.03 + rank 50: 78764.12 + rank 51: 78746.13 + rank 52: 78767.34 + rank 53: 78756.12 + rank 54: 78684.07 + rank 55: 78753.58 + rank 56: 78716.15 + rank 57: 78700.68 + rank 58: 78704.19 + rank 59: 78721.26 + rank 60: 78731.26 + rank 61: 78738.10 + rank 62: 78693.24 + rank 63: 78722.01 + backward-compute: + rank 0: 118599.38 + rank 1: 118587.30 + rank 2: 118577.16 + rank 3: 118581.97 + rank 4: 118566.14 + rank 5: 118590.41 + rank 6: 118574.68 + rank 7: 118504.33 + rank 8: 118551.10 + rank 9: 118582.64 + rank 10: 118584.24 + rank 11: 118569.05 + rank 12: 118558.52 + rank 13: 118544.49 + rank 14: 118576.75 + rank 15: 118578.80 + rank 16: 74897.76 + rank 17: 74895.44 + rank 18: 74896.03 + rank 19: 74888.31 + rank 20: 74888.63 + rank 21: 74894.11 + rank 22: 74897.16 + rank 23: 74887.60 + rank 24: 74871.17 + rank 25: 74879.42 + rank 26: 74874.52 + rank 27: 74876.23 + rank 28: 74868.14 + rank 29: 74872.03 + rank 30: 74870.36 + rank 31: 74870.77 + rank 32: 88855.67 + rank 33: 88904.74 + rank 34: 88905.18 + rank 35: 88908.33 + rank 36: 88898.63 + rank 37: 88905.30 + rank 38: 88899.26 + rank 39: 88902.45 + rank 40: 88908.90 + rank 41: 88916.13 + rank 42: 88915.45 + rank 43: 88913.21 + rank 44: 88917.50 + rank 45: 88912.88 + rank 46: 88916.62 + rank 47: 88919.78 + rank 48: 106720.12 + rank 49: 106724.32 + rank 50: 106721.01 + rank 51: 106738.70 + rank 52: 106712.09 + rank 53: 106728.34 + rank 54: 106800.26 + rank 55: 106729.93 + rank 56: 106768.44 + rank 57: 106784.36 + rank 58: 106780.73 + rank 59: 106762.46 + rank 60: 106751.54 + rank 61: 106743.67 + rank 62: 106793.48 + rank 63: 106763.44 + batch-generator: + rank 0: 15.34 + rank 1: 15.20 + rank 2: 14.79 + rank 3: 14.70 + rank 4: 18.29 + rank 5: 16.01 + rank 6: 15.14 + rank 7: 15.76 + rank 8: 20.05 + rank 9: 16.22 + rank 10: 18.62 + rank 11: 17.95 + rank 12: 24.15 + rank 13: 22.84 + rank 14: 17.49 + rank 15: 15.60 + rank 16: 16.13 + rank 17: 15.60 + rank 18: 15.16 + rank 19: 15.63 + rank 20: 16.94 + rank 21: 16.40 + rank 22: 15.27 + rank 23: 15.09 + rank 24: 22.54 + rank 25: 21.91 + rank 26: 23.66 + rank 27: 23.21 + rank 28: 20.50 + rank 29: 19.59 + rank 30: 20.28 + rank 31: 20.92 + rank 32: 20.95 + rank 33: 22.13 + rank 34: 22.65 + rank 35: 21.85 + rank 36: 22.66 + rank 37: 25.52 + rank 38: 24.35 + rank 39: 24.65 + rank 40: 16.11 + rank 41: 15.37 + rank 42: 15.70 + rank 43: 15.39 + rank 44: 16.13 + rank 45: 15.52 + rank 46: 15.43 + rank 47: 15.23 + rank 48: 20.02 + rank 49: 16.91 + rank 50: 15.50 + rank 51: 17.58 + rank 52: 19.03 + rank 53: 19.34 + rank 54: 16.83 + rank 55: 19.76 + rank 56: 15.61 + rank 57: 15.02 + rank 58: 15.62 + rank 59: 14.51 + rank 60: 14.12 + rank 61: 15.76 + rank 62: 14.83 + rank 63: 15.19 + embedding-grads-all-reduce: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.03 + rank 3: 0.02 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.03 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.03 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.02 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.02 + rank 48: 0.04 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.04 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.03 + rank 62: 0.02 + rank 63: 0.02 + all-grads-sync: + rank 0: 1.06 + rank 1: 0.78 + rank 2: 0.79 + rank 3: 1.07 + rank 4: 1.64 + rank 5: 1.16 + rank 6: 0.75 + rank 7: 0.90 + rank 8: 0.95 + rank 9: 0.95 + rank 10: 1.13 + rank 11: 0.78 + rank 12: 1.16 + rank 13: 0.79 + rank 14: 1.10 + rank 15: 0.86 + rank 16: 0.76 + rank 17: 0.83 + rank 18: 0.77 + rank 19: 1.01 + rank 20: 0.81 + rank 21: 0.77 + rank 22: 0.74 + rank 23: 0.89 + rank 24: 1.25 + rank 25: 1.32 + rank 26: 1.26 + rank 27: 1.20 + rank 28: 1.15 + rank 29: 1.21 + rank 30: 1.17 + rank 31: 1.12 + rank 32: 1.19 + rank 33: 1.46 + rank 34: 0.99 + rank 35: 1.40 + rank 36: 1.22 + rank 37: 1.46 + rank 38: 1.44 + rank 39: 1.25 + rank 40: 0.80 + rank 41: 0.83 + rank 42: 0.81 + rank 43: 0.77 + rank 44: 0.97 + rank 45: 1.03 + rank 46: 0.79 + rank 47: 0.80 + rank 48: 1.38 + rank 49: 1.20 + rank 50: 0.91 + rank 51: 0.81 + rank 52: 0.99 + rank 53: 1.10 + rank 54: 0.98 + rank 55: 1.11 + rank 56: 1.06 + rank 57: 0.87 + rank 58: 0.95 + rank 59: 0.77 + rank 60: 0.86 + rank 61: 1.12 + rank 62: 0.80 + rank 63: 0.87 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.02 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.02 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.03 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.14 + rank 1: 1.77 + rank 2: 1.77 + rank 3: 2.13 + rank 4: 1.97 + rank 5: 1.91 + rank 6: 1.88 + rank 7: 1.99 + rank 8: 1.82 + rank 9: 2.20 + rank 10: 1.81 + rank 11: 1.80 + rank 12: 1.94 + rank 13: 1.81 + rank 14: 1.97 + rank 15: 2.02 + rank 16: 2.40 + rank 17: 2.15 + rank 18: 2.12 + rank 19: 2.17 + rank 20: 2.22 + rank 21: 2.25 + rank 22: 2.53 + rank 23: 2.25 + rank 24: 2.02 + rank 25: 2.30 + rank 26: 2.06 + rank 27: 2.13 + rank 28: 2.24 + rank 29: 2.33 + rank 30: 2.13 + rank 31: 2.04 + rank 32: 2.12 + rank 33: 2.00 + rank 34: 2.06 + rank 35: 2.29 + rank 36: 2.17 + rank 37: 2.14 + rank 38: 2.47 + rank 39: 1.99 + rank 40: 1.78 + rank 41: 1.89 + rank 42: 1.86 + rank 43: 1.87 + rank 44: 1.90 + rank 45: 2.04 + rank 46: 1.79 + rank 47: 1.73 + rank 48: 2.33 + rank 49: 1.81 + rank 50: 2.42 + rank 51: 1.94 + rank 52: 2.01 + rank 53: 1.92 + rank 54: 1.84 + rank 55: 2.18 + rank 56: 1.83 + rank 57: 2.42 + rank 58: 1.83 + rank 59: 1.80 + rank 60: 1.73 + rank 61: 1.76 + rank 62: 1.91 + rank 63: 1.78 + optimizer-inner-step: + rank 0: 41.97 + rank 1: 41.81 + rank 2: 41.34 + rank 3: 41.67 + rank 4: 41.41 + rank 5: 41.55 + rank 6: 41.94 + rank 7: 41.82 + rank 8: 41.68 + rank 9: 41.58 + rank 10: 41.54 + rank 11: 41.68 + rank 12: 41.69 + rank 13: 41.88 + rank 14: 41.64 + rank 15: 41.55 + rank 16: 42.11 + rank 17: 41.77 + rank 18: 41.97 + rank 19: 41.72 + rank 20: 41.83 + rank 21: 41.76 + rank 22: 42.14 + rank 23: 41.91 + rank 24: 41.50 + rank 25: 41.97 + rank 26: 41.69 + rank 27: 42.31 + rank 28: 46.31 + rank 29: 41.41 + rank 30: 41.58 + rank 31: 42.09 + rank 32: 41.71 + rank 33: 41.50 + rank 34: 41.60 + rank 35: 48.76 + rank 36: 42.55 + rank 37: 41.71 + rank 38: 41.60 + rank 39: 41.57 + rank 40: 41.67 + rank 41: 41.74 + rank 42: 41.89 + rank 43: 41.92 + rank 44: 41.41 + rank 45: 44.22 + rank 46: 41.78 + rank 47: 41.36 + rank 48: 43.92 + rank 49: 41.57 + rank 50: 53.32 + rank 51: 41.43 + rank 52: 41.68 + rank 53: 41.64 + rank 54: 41.69 + rank 55: 41.65 + rank 56: 41.34 + rank 57: 47.21 + rank 58: 41.69 + rank 59: 41.43 + rank 60: 41.89 + rank 61: 41.46 + rank 62: 41.58 + rank 63: 41.48 + optimizer-copy-main-to-model-params: + rank 0: 4.50 + rank 1: 3.93 + rank 2: 3.83 + rank 3: 4.40 + rank 4: 3.83 + rank 5: 4.02 + rank 6: 4.02 + rank 7: 4.07 + rank 8: 3.92 + rank 9: 4.48 + rank 10: 3.89 + rank 11: 3.88 + rank 12: 4.08 + rank 13: 3.88 + rank 14: 4.01 + rank 15: 4.25 + rank 16: 4.43 + rank 17: 4.11 + rank 18: 4.07 + rank 19: 4.10 + rank 20: 4.14 + rank 21: 4.32 + rank 22: 4.72 + rank 23: 4.26 + rank 24: 4.13 + rank 25: 4.35 + rank 26: 4.15 + rank 27: 4.21 + rank 28: 4.50 + rank 29: 4.50 + rank 30: 4.40 + rank 31: 4.33 + rank 32: 4.23 + rank 33: 4.06 + rank 34: 4.28 + rank 35: 4.74 + rank 36: 4.41 + rank 37: 4.24 + rank 38: 4.80 + rank 39: 4.02 + rank 40: 3.82 + rank 41: 4.01 + rank 42: 3.96 + rank 43: 4.03 + rank 44: 4.05 + rank 45: 4.37 + rank 46: 3.88 + rank 47: 3.74 + rank 48: 4.37 + rank 49: 4.02 + rank 50: 5.08 + rank 51: 4.22 + rank 52: 4.31 + rank 53: 4.04 + rank 54: 4.00 + rank 55: 4.16 + rank 56: 3.93 + rank 57: 4.82 + rank 58: 4.00 + rank 59: 3.82 + rank 60: 3.85 + rank 61: 3.88 + rank 62: 3.96 + rank 63: 3.84 + optimizer: + rank 0: 53.64 + rank 1: 52.73 + rank 2: 52.13 + rank 3: 53.22 + rank 4: 52.14 + rank 5: 52.55 + rank 6: 52.96 + rank 7: 52.88 + rank 8: 52.52 + rank 9: 53.28 + rank 10: 52.30 + rank 11: 52.45 + rank 12: 52.66 + rank 13: 52.68 + rank 14: 52.61 + rank 15: 52.84 + rank 16: 53.95 + rank 17: 53.03 + rank 18: 53.17 + rank 19: 52.99 + rank 20: 53.12 + rank 21: 53.26 + rank 22: 54.29 + rank 23: 53.39 + rank 24: 52.56 + rank 25: 53.44 + rank 26: 52.79 + rank 27: 53.60 + rank 28: 58.00 + rank 29: 53.10 + rank 30: 53.17 + rank 31: 53.43 + rank 32: 52.85 + rank 33: 52.41 + rank 34: 52.79 + rank 35: 60.57 + rank 36: 53.94 + rank 37: 52.87 + rank 38: 53.48 + rank 39: 52.43 + rank 40: 52.36 + rank 41: 52.70 + rank 42: 52.81 + rank 43: 52.91 + rank 44: 52.43 + rank 45: 55.74 + rank 46: 52.53 + rank 47: 51.96 + rank 48: 55.27 + rank 49: 52.46 + rank 50: 65.67 + rank 51: 52.56 + rank 52: 52.97 + rank 53: 52.56 + rank 54: 52.59 + rank 55: 52.70 + rank 56: 52.18 + rank 57: 59.29 + rank 58: 52.64 + rank 59: 52.16 + rank 60: 52.64 + rank 61: 52.23 + rank 62: 52.50 + rank 63: 52.20 + interval-time: + rank 0: 4.09 + rank 1: 4.06 + rank 2: 4.08 + rank 3: 4.07 + rank 4: 4.16 + rank 5: 4.08 + rank 6: 4.06 + rank 7: 4.07 + rank 8: 0.79 + rank 9: 0.69 + rank 10: 0.70 + rank 11: 0.70 + rank 12: 0.71 + rank 13: 0.77 + rank 14: 0.69 + rank 15: 0.68 + rank 16: 0.72 + rank 17: 0.70 + rank 18: 0.69 + rank 19: 0.78 + rank 20: 0.86 + rank 21: 0.71 + rank 22: 0.69 + rank 23: 0.69 + rank 24: 1.20 + rank 25: 1.01 + rank 26: 1.17 + rank 27: 1.08 + rank 28: 1.16 + rank 29: 1.07 + rank 30: 1.19 + rank 31: 1.56 + rank 32: 0.80 + rank 33: 0.81 + rank 34: 0.78 + rank 35: 0.77 + rank 36: 0.79 + rank 37: 0.79 + rank 38: 0.94 + rank 39: 0.81 + rank 40: 0.70 + rank 41: 0.71 + rank 42: 0.91 + rank 43: 0.80 + rank 44: 0.72 + rank 45: 0.79 + rank 46: 0.72 + rank 47: 0.74 + rank 48: 0.83 + rank 49: 0.76 + rank 50: 0.77 + rank 51: 0.77 + rank 52: 0.87 + rank 53: 0.76 + rank 54: 0.86 + rank 55: 0.90 + rank 56: 0.69 + rank 57: 0.69 + rank 58: 0.68 + rank 59: 0.69 + rank 60: 0.69 + rank 61: 0.78 + rank 62: 0.78 + rank 63: 2.35 + [2026-10-11 03:06:07] iteration 4/ 2384 | consumed samples: 128 | elapsed time per iteration (ms): 241053.4 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 69599.4 | valid tokens/s: 69599.4 | physical tokens/s: 69881.7 | loss tokens/s: 54766.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.709281E-01 | lm-global-supervised-tokens: 1.320158E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.434451E-02 | lm-token-buffer-mib: 1.162232E+02 | global_load_balancing_loss: 1.000207E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11236.030 | params rms: 0.060839 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:21:47.098336 | finish at 2026-10-17 18:27:54 +times across ranks (ms): + forward-backward: + rank 0: 240830.55 + rank 1: 240831.47 + rank 2: 240832.45 + rank 3: 240832.48 + rank 4: 240833.78 + rank 5: 240832.53 + rank 6: 240829.36 + rank 7: 240831.34 + rank 8: 240820.22 + rank 9: 240822.02 + rank 10: 240821.23 + rank 11: 240822.03 + rank 12: 240823.34 + rank 13: 240823.02 + rank 14: 240820.22 + rank 15: 240819.53 + rank 16: 240824.23 + rank 17: 240825.52 + rank 18: 240826.73 + rank 19: 240826.81 + rank 20: 240827.22 + rank 21: 240828.59 + rank 22: 240823.41 + rank 23: 240824.69 + rank 24: 240811.97 + rank 25: 240814.34 + rank 26: 240814.41 + rank 27: 240814.06 + rank 28: 240816.50 + rank 29: 240814.62 + rank 30: 240813.59 + rank 31: 240811.12 + rank 32: 240817.16 + rank 33: 240817.80 + rank 34: 240819.16 + rank 35: 240817.80 + rank 36: 240818.97 + rank 37: 240818.98 + rank 38: 240817.12 + rank 39: 240819.91 + rank 40: 240812.42 + rank 41: 240812.56 + rank 42: 240812.20 + rank 43: 240811.14 + rank 44: 240811.91 + rank 45: 240809.94 + rank 46: 240814.08 + rank 47: 240813.03 + rank 48: 240833.80 + rank 49: 240833.55 + rank 50: 240832.73 + rank 51: 240833.45 + rank 52: 240831.36 + rank 53: 240832.66 + rank 54: 240833.28 + rank 55: 240836.41 + rank 56: 240826.02 + rank 57: 240826.56 + rank 58: 240825.50 + rank 59: 240825.05 + rank 60: 240825.47 + rank 61: 240823.77 + rank 62: 240827.58 + rank 63: 240826.38 + forward-compute: + rank 0: 157310.20 + rank 1: 157307.75 + rank 2: 157319.95 + rank 3: 157314.64 + rank 4: 157310.78 + rank 5: 157304.98 + rank 6: 157308.77 + rank 7: 157318.28 + rank 8: 157307.09 + rank 9: 157302.19 + rank 10: 157305.62 + rank 11: 157311.36 + rank 12: 157313.03 + rank 13: 157331.86 + rank 14: 157323.08 + rank 15: 157311.92 + rank 16: 88663.73 + rank 17: 88651.62 + rank 18: 88677.80 + rank 19: 88683.73 + rank 20: 88675.74 + rank 21: 88670.82 + rank 22: 88664.66 + rank 23: 88684.75 + rank 24: 88642.18 + rank 25: 88646.71 + rank 26: 88643.34 + rank 27: 88640.23 + rank 28: 88670.38 + rank 29: 88666.62 + rank 30: 88665.85 + rank 31: 88653.16 + rank 32: 168756.28 + rank 33: 168761.91 + rank 34: 168766.36 + rank 35: 168767.94 + rank 36: 168768.20 + rank 37: 168771.66 + rank 38: 168773.58 + rank 39: 168768.61 + rank 40: 168758.25 + rank 41: 168763.08 + rank 42: 168763.42 + rank 43: 168762.69 + rank 44: 168760.23 + rank 45: 168763.56 + rank 46: 168764.58 + rank 47: 168757.84 + rank 48: 140536.75 + rank 49: 140564.59 + rank 50: 140546.94 + rank 51: 140566.45 + rank 52: 140588.16 + rank 53: 140579.52 + rank 54: 140512.98 + rank 55: 140576.44 + rank 56: 140531.36 + rank 57: 140522.75 + rank 58: 140532.14 + rank 59: 140525.14 + rank 60: 140522.00 + rank 61: 140550.84 + rank 62: 140532.05 + rank 63: 140540.97 + backward-compute: + rank 0: 83488.33 + rank 1: 83489.45 + rank 2: 83475.47 + rank 3: 83482.95 + rank 4: 83475.45 + rank 5: 83492.43 + rank 6: 83486.25 + rank 7: 83479.24 + rank 8: 83482.05 + rank 9: 83489.43 + rank 10: 83480.15 + rank 11: 83478.70 + rank 12: 83473.21 + rank 13: 83459.63 + rank 14: 83465.88 + rank 15: 83477.15 + rank 16: 152129.03 + rank 17: 152149.92 + rank 18: 152123.20 + rank 19: 152118.42 + rank 20: 152125.78 + rank 21: 152130.61 + rank 22: 152133.91 + rank 23: 152115.70 + rank 24: 152132.97 + rank 25: 152129.67 + rank 26: 152137.03 + rank 27: 152136.03 + rank 28: 152110.83 + rank 29: 152116.66 + rank 30: 152114.83 + rank 31: 152121.38 + rank 32: 72025.98 + rank 33: 72020.78 + rank 34: 72019.23 + rank 35: 72015.49 + rank 36: 72015.73 + rank 37: 72011.48 + rank 38: 71999.44 + rank 39: 72014.43 + rank 40: 72024.78 + rank 41: 72018.60 + rank 42: 72015.77 + rank 43: 72019.00 + rank 44: 72021.46 + rank 45: 72014.95 + rank 46: 72015.95 + rank 47: 72022.92 + rank 48: 100265.34 + rank 49: 100240.34 + rank 50: 100256.91 + rank 51: 100238.68 + rank 52: 100214.52 + rank 53: 100225.80 + rank 54: 100291.22 + rank 55: 100229.99 + rank 56: 100266.46 + rank 57: 100275.05 + rank 58: 100265.10 + rank 59: 100272.41 + rank 60: 100276.50 + rank 61: 100246.86 + rank 62: 100268.35 + rank 63: 100257.70 + batch-generator: + rank 0: 15.03 + rank 1: 14.49 + rank 2: 15.57 + rank 3: 16.06 + rank 4: 17.18 + rank 5: 15.29 + rank 6: 16.02 + rank 7: 16.45 + rank 8: 16.09 + rank 9: 15.89 + rank 10: 15.57 + rank 11: 16.97 + rank 12: 18.65 + rank 13: 18.85 + rank 14: 16.15 + rank 15: 15.32 + rank 16: 17.23 + rank 17: 15.41 + rank 18: 17.02 + rank 19: 16.12 + rank 20: 17.05 + rank 21: 17.33 + rank 22: 16.79 + rank 23: 16.12 + rank 24: 24.89 + rank 25: 23.15 + rank 26: 25.67 + rank 27: 22.94 + rank 28: 21.02 + rank 29: 20.03 + rank 30: 23.58 + rank 31: 24.47 + rank 32: 23.31 + rank 33: 23.33 + rank 34: 18.95 + rank 35: 21.55 + rank 36: 22.11 + rank 37: 22.81 + rank 38: 23.09 + rank 39: 23.39 + rank 40: 14.88 + rank 41: 14.42 + rank 42: 16.60 + rank 43: 17.21 + rank 44: 14.72 + rank 45: 15.39 + rank 46: 14.90 + rank 47: 14.62 + rank 48: 19.14 + rank 49: 17.64 + rank 50: 16.04 + rank 51: 18.28 + rank 52: 17.91 + rank 53: 17.65 + rank 54: 15.80 + rank 55: 17.24 + rank 56: 14.42 + rank 57: 14.06 + rank 58: 15.61 + rank 59: 16.11 + rank 60: 14.70 + rank 61: 14.53 + rank 62: 14.33 + rank 63: 14.45 + embedding-grads-all-reduce: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.04 + rank 17: 0.02 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.02 + rank 48: 0.06 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.02 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.02 + all-grads-sync: + rank 0: 1.01 + rank 1: 0.77 + rank 2: 0.78 + rank 3: 0.81 + rank 4: 1.48 + rank 5: 0.81 + rank 6: 0.94 + rank 7: 1.15 + rank 8: 0.75 + rank 9: 0.82 + rank 10: 0.75 + rank 11: 1.00 + rank 12: 0.83 + rank 13: 0.80 + rank 14: 0.78 + rank 15: 0.81 + rank 16: 0.79 + rank 17: 0.91 + rank 18: 0.80 + rank 19: 1.00 + rank 20: 1.05 + rank 21: 1.15 + rank 22: 0.95 + rank 23: 0.87 + rank 24: 1.34 + rank 25: 1.21 + rank 26: 1.22 + rank 27: 1.40 + rank 28: 1.31 + rank 29: 1.27 + rank 30: 1.59 + rank 31: 1.31 + rank 32: 1.43 + rank 33: 1.20 + rank 34: 1.86 + rank 35: 1.21 + rank 36: 1.20 + rank 37: 1.10 + rank 38: 1.06 + rank 39: 1.29 + rank 40: 0.91 + rank 41: 0.84 + rank 42: 0.80 + rank 43: 0.88 + rank 44: 0.88 + rank 45: 0.85 + rank 46: 0.82 + rank 47: 1.12 + rank 48: 1.13 + rank 49: 1.01 + rank 50: 1.14 + rank 51: 1.24 + rank 52: 0.92 + rank 53: 1.30 + rank 54: 0.94 + rank 55: 1.74 + rank 56: 0.77 + rank 57: 0.79 + rank 58: 0.82 + rank 59: 0.80 + rank 60: 0.98 + rank 61: 0.85 + rank 62: 0.79 + rank 63: 0.76 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.04 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.02 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.11 + rank 1: 1.80 + rank 2: 1.79 + rank 3: 2.20 + rank 4: 1.91 + rank 5: 1.91 + rank 6: 1.85 + rank 7: 1.95 + rank 8: 1.87 + rank 9: 2.24 + rank 10: 1.78 + rank 11: 1.80 + rank 12: 1.91 + rank 13: 1.82 + rank 14: 1.95 + rank 15: 2.03 + rank 16: 4.25 + rank 17: 3.23 + rank 18: 3.18 + rank 19: 3.29 + rank 20: 3.04 + rank 21: 3.24 + rank 22: 3.82 + rank 23: 3.49 + rank 24: 1.92 + rank 25: 2.25 + rank 26: 2.05 + rank 27: 2.05 + rank 28: 2.30 + rank 29: 2.23 + rank 30: 2.33 + rank 31: 2.02 + rank 32: 2.04 + rank 33: 1.92 + rank 34: 2.00 + rank 35: 2.23 + rank 36: 2.26 + rank 37: 2.08 + rank 38: 2.27 + rank 39: 1.97 + rank 40: 1.73 + rank 41: 1.82 + rank 42: 1.88 + rank 43: 1.88 + rank 44: 1.85 + rank 45: 2.10 + rank 46: 1.79 + rank 47: 1.78 + rank 48: 2.90 + rank 49: 1.81 + rank 50: 2.39 + rank 51: 1.92 + rank 52: 2.02 + rank 53: 1.88 + rank 54: 1.82 + rank 55: 1.97 + rank 56: 1.81 + rank 57: 2.46 + rank 58: 1.86 + rank 59: 1.82 + rank 60: 1.78 + rank 61: 1.76 + rank 62: 1.92 + rank 63: 1.76 + optimizer-inner-step: + rank 0: 42.04 + rank 1: 41.89 + rank 2: 41.60 + rank 3: 41.83 + rank 4: 41.55 + rank 5: 41.64 + rank 6: 42.05 + rank 7: 41.97 + rank 8: 41.73 + rank 9: 41.83 + rank 10: 41.65 + rank 11: 41.71 + rank 12: 41.72 + rank 13: 41.97 + rank 14: 41.70 + rank 15: 41.64 + rank 16: 42.00 + rank 17: 41.84 + rank 18: 41.87 + rank 19: 41.73 + rank 20: 41.80 + rank 21: 41.76 + rank 22: 42.21 + rank 23: 41.92 + rank 24: 41.55 + rank 25: 41.78 + rank 26: 41.77 + rank 27: 42.76 + rank 28: 46.16 + rank 29: 41.62 + rank 30: 41.64 + rank 31: 41.87 + rank 32: 41.78 + rank 33: 41.57 + rank 34: 41.77 + rank 35: 47.85 + rank 36: 42.33 + rank 37: 41.85 + rank 38: 41.67 + rank 39: 41.65 + rank 40: 41.74 + rank 41: 41.80 + rank 42: 41.94 + rank 43: 42.00 + rank 44: 41.48 + rank 45: 42.14 + rank 46: 41.88 + rank 47: 41.47 + rank 48: 43.92 + rank 49: 41.65 + rank 50: 53.31 + rank 51: 41.41 + rank 52: 41.73 + rank 53: 41.68 + rank 54: 41.80 + rank 55: 41.69 + rank 56: 41.36 + rank 57: 47.54 + rank 58: 41.72 + rank 59: 41.56 + rank 60: 41.90 + rank 61: 41.51 + rank 62: 41.64 + rank 63: 41.49 + optimizer-copy-main-to-model-params: + rank 0: 4.42 + rank 1: 3.85 + rank 2: 3.84 + rank 3: 4.42 + rank 4: 3.81 + rank 5: 3.96 + rank 6: 3.98 + rank 7: 4.02 + rank 8: 3.95 + rank 9: 4.52 + rank 10: 3.88 + rank 11: 3.86 + rank 12: 4.03 + rank 13: 3.83 + rank 14: 4.09 + rank 15: 4.28 + rank 16: 4.55 + rank 17: 4.16 + rank 18: 4.17 + rank 19: 4.13 + rank 20: 4.18 + rank 21: 4.34 + rank 22: 4.78 + rank 23: 4.32 + rank 24: 4.02 + rank 25: 4.40 + rank 26: 4.10 + rank 27: 4.27 + rank 28: 4.52 + rank 29: 4.49 + rank 30: 4.44 + rank 31: 4.23 + rank 32: 4.28 + rank 33: 4.08 + rank 34: 4.40 + rank 35: 4.68 + rank 36: 4.51 + rank 37: 4.20 + rank 38: 4.64 + rank 39: 4.07 + rank 40: 3.78 + rank 41: 4.04 + rank 42: 3.99 + rank 43: 4.01 + rank 44: 4.03 + rank 45: 4.41 + rank 46: 3.83 + rank 47: 3.74 + rank 48: 4.37 + rank 49: 4.02 + rank 50: 5.06 + rank 51: 4.12 + rank 52: 4.31 + rank 53: 4.02 + rank 54: 3.97 + rank 55: 4.04 + rank 56: 3.85 + rank 57: 4.86 + rank 58: 3.95 + rank 59: 3.83 + rank 60: 3.88 + rank 61: 3.77 + rank 62: 3.97 + rank 63: 3.74 + optimizer: + rank 0: 56.31 + rank 1: 55.39 + rank 2: 55.03 + rank 3: 56.07 + rank 4: 54.96 + rank 5: 55.27 + rank 6: 55.70 + rank 7: 55.66 + rank 8: 55.33 + rank 9: 56.20 + rank 10: 55.14 + rank 11: 55.19 + rank 12: 55.41 + rank 13: 55.44 + rank 14: 55.50 + rank 15: 55.67 + rank 16: 56.76 + rank 17: 55.81 + rank 18: 56.02 + rank 19: 55.79 + rank 20: 55.90 + rank 21: 56.04 + rank 22: 57.06 + rank 23: 56.15 + rank 24: 55.15 + rank 25: 55.89 + rank 26: 55.51 + rank 27: 56.68 + rank 28: 60.47 + rank 29: 55.85 + rank 30: 55.81 + rank 31: 55.73 + rank 32: 55.69 + rank 33: 55.19 + rank 34: 55.77 + rank 35: 62.31 + rank 36: 56.52 + rank 37: 55.66 + rank 38: 56.06 + rank 39: 55.27 + rank 40: 55.08 + rank 41: 55.51 + rank 42: 55.57 + rank 43: 55.65 + rank 44: 55.18 + rank 45: 56.35 + rank 46: 55.26 + rank 47: 54.79 + rank 48: 57.99 + rank 49: 55.26 + rank 50: 68.35 + rank 51: 55.15 + rank 52: 55.75 + rank 53: 55.30 + rank 54: 55.35 + rank 55: 55.33 + rank 56: 54.82 + rank 57: 62.36 + rank 58: 55.30 + rank 59: 54.98 + rank 60: 55.40 + rank 61: 54.86 + rank 62: 55.28 + rank 63: 54.79 + interval-time: + rank 0: 0.68 + rank 1: 0.67 + rank 2: 0.70 + rank 3: 0.69 + rank 4: 0.84 + rank 5: 0.69 + rank 6: 0.68 + rank 7: 0.68 + rank 8: 0.77 + rank 9: 0.70 + rank 10: 0.71 + rank 11: 0.70 + rank 12: 0.69 + rank 13: 0.77 + rank 14: 0.70 + rank 15: 0.69 + rank 16: 0.78 + rank 17: 0.69 + rank 18: 0.69 + rank 19: 0.70 + rank 20: 0.82 + rank 21: 0.72 + rank 22: 0.68 + rank 23: 0.78 + rank 24: 1.01 + rank 25: 0.78 + rank 26: 0.95 + rank 27: 0.93 + rank 28: 0.87 + rank 29: 0.77 + rank 30: 0.79 + rank 31: 0.87 + rank 32: 0.78 + rank 33: 0.88 + rank 34: 0.77 + rank 35: 0.76 + rank 36: 0.78 + rank 37: 0.79 + rank 38: 0.86 + rank 39: 0.78 + rank 40: 0.71 + rank 41: 0.71 + rank 42: 0.79 + rank 43: 0.79 + rank 44: 0.71 + rank 45: 0.80 + rank 46: 0.81 + rank 47: 0.78 + rank 48: 0.89 + rank 49: 0.79 + rank 50: 0.79 + rank 51: 0.79 + rank 52: 0.87 + rank 53: 0.77 + rank 54: 0.88 + rank 55: 0.88 + rank 56: 0.70 + rank 57: 0.70 + rank 58: 0.71 + rank 59: 0.80 + rank 60: 0.70 + rank 61: 0.77 + rank 62: 0.79 + rank 63: 2.13 + [2026-10-11 03:08:57] iteration 5/ 2384 | consumed samples: 160 | elapsed time per iteration (ms): 170225.9 | throughput per GPU (TFLOP/s/GPU): 31.5 | MFU 10.11% | FLOPs estimator: workload-aware | logical tokens/s: 98558.2 | valid tokens/s: 98558.2 | physical tokens/s: 99051.7 | loss tokens/s: 75081.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.093625E-01 | lm-global-supervised-tokens: 1.278085E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.620786E-02 | lm-token-buffer-mib: 1.170197E+02 | global_load_balancing_loss: 1.001644E+00 | loss scale: 1.0 | grad norm: 0.290 | params norm: 11235.979 | params rms: 0.060839 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:29:27.307043 | finish at 2026-10-15 19:38:25 +times across ranks (ms): + forward-backward: + rank 0: 170004.28 + rank 1: 170006.31 + rank 2: 170005.09 + rank 3: 170006.16 + rank 4: 170007.66 + rank 5: 170007.44 + rank 6: 170004.41 + rank 7: 170004.88 + rank 8: 170000.28 + rank 9: 170003.30 + rank 10: 170003.42 + rank 11: 170002.84 + rank 12: 170003.48 + rank 13: 170004.30 + rank 14: 170000.98 + rank 15: 170001.91 + rank 16: 169997.41 + rank 17: 169999.12 + rank 18: 169998.08 + rank 19: 170000.09 + rank 20: 169999.41 + rank 21: 170001.44 + rank 22: 169996.64 + rank 23: 169997.77 + rank 24: 169992.75 + rank 25: 169995.27 + rank 26: 169995.95 + rank 27: 169996.00 + rank 28: 169997.48 + rank 29: 169997.16 + rank 30: 169993.00 + rank 31: 169993.33 + rank 32: 169990.11 + rank 33: 169990.52 + rank 34: 169989.34 + rank 35: 169990.78 + rank 36: 169991.22 + rank 37: 169991.86 + rank 38: 169991.16 + rank 39: 169992.45 + rank 40: 169993.38 + rank 41: 169994.73 + rank 42: 169994.19 + rank 43: 169992.39 + rank 44: 169992.00 + rank 45: 169991.42 + rank 46: 169995.55 + rank 47: 169995.91 + rank 48: 170004.39 + rank 49: 170004.86 + rank 50: 170003.22 + rank 51: 170007.02 + rank 52: 170001.39 + rank 53: 170004.30 + rank 54: 170005.12 + rank 55: 170007.50 + rank 56: 170006.02 + rank 57: 170006.73 + rank 58: 170006.53 + rank 59: 170005.23 + rank 60: 170004.97 + rank 61: 170004.45 + rank 62: 170007.28 + rank 63: 170008.27 + forward-compute: + rank 0: 88038.78 + rank 1: 88050.40 + rank 2: 88053.67 + rank 3: 88055.92 + rank 4: 87988.14 + rank 5: 88051.88 + rank 6: 88055.20 + rank 7: 88000.08 + rank 8: 88054.30 + rank 9: 88054.92 + rank 10: 87995.34 + rank 11: 88072.97 + rank 12: 88070.27 + rank 13: 88075.16 + rank 14: 87989.48 + rank 15: 87990.29 + rank 16: 86214.58 + rank 17: 86216.48 + rank 18: 86217.27 + rank 19: 86226.00 + rank 20: 86226.36 + rank 21: 86229.33 + rank 22: 86225.12 + rank 23: 86233.45 + rank 24: 86222.06 + rank 25: 86219.61 + rank 26: 86226.75 + rank 27: 86221.14 + rank 28: 86225.12 + rank 29: 86221.31 + rank 30: 86212.48 + rank 31: 86212.33 + rank 32: 79899.79 + rank 33: 79908.41 + rank 34: 79901.15 + rank 35: 79904.23 + rank 36: 79913.66 + rank 37: 79925.49 + rank 38: 79922.77 + rank 39: 79900.88 + rank 40: 79909.47 + rank 41: 79902.80 + rank 42: 79901.74 + rank 43: 79898.74 + rank 44: 79902.49 + rank 45: 79908.48 + rank 46: 79906.70 + rank 47: 79909.16 + rank 48: 55019.90 + rank 49: 55030.86 + rank 50: 55032.17 + rank 51: 55021.45 + rank 52: 55065.94 + rank 53: 55072.37 + rank 54: 55008.91 + rank 55: 55045.09 + rank 56: 55022.09 + rank 57: 55042.91 + rank 58: 55040.57 + rank 59: 55030.01 + rank 60: 55019.71 + rank 61: 55055.73 + rank 62: 55026.59 + rank 63: 55043.15 + backward-compute: + rank 0: 81937.06 + rank 1: 81928.60 + rank 2: 81923.38 + rank 3: 81923.78 + rank 4: 81968.49 + rank 5: 81929.18 + rank 6: 81922.21 + rank 7: 81977.59 + rank 8: 81916.60 + rank 9: 81918.37 + rank 10: 81968.70 + rank 11: 81902.07 + rank 12: 81905.98 + rank 13: 81901.62 + rank 14: 81982.03 + rank 15: 81985.17 + rank 16: 83750.16 + rank 17: 83753.47 + rank 18: 83753.60 + rank 19: 83747.45 + rank 20: 83746.38 + rank 21: 83742.55 + rank 22: 83744.76 + rank 23: 83735.45 + rank 24: 83732.53 + rank 25: 83732.73 + rank 26: 83732.27 + rank 27: 83733.39 + rank 28: 83731.45 + rank 29: 83744.75 + rank 30: 83743.97 + rank 31: 83744.41 + rank 32: 90061.15 + rank 33: 90051.42 + rank 34: 90057.98 + rank 35: 90056.48 + rank 36: 90046.48 + rank 37: 90036.72 + rank 38: 90038.34 + rank 39: 90061.72 + rank 40: 90051.53 + rank 41: 90064.73 + rank 42: 90066.56 + rank 43: 90067.00 + rank 44: 90058.41 + rank 45: 90057.90 + rank 46: 90061.18 + rank 47: 90062.26 + rank 48: 114957.76 + rank 49: 114947.55 + rank 50: 114945.89 + rank 51: 114954.34 + rank 52: 114911.68 + rank 53: 114907.76 + rank 54: 114971.48 + rank 55: 114936.29 + rank 56: 114959.74 + rank 57: 114940.54 + rank 58: 114942.66 + rank 59: 114950.70 + rank 60: 114961.56 + rank 61: 114924.88 + rank 62: 114957.52 + rank 63: 114937.84 + batch-generator: + rank 0: 15.92 + rank 1: 16.76 + rank 2: 15.96 + rank 3: 16.13 + rank 4: 17.58 + rank 5: 14.38 + rank 6: 15.80 + rank 7: 16.07 + rank 8: 15.60 + rank 9: 15.84 + rank 10: 16.72 + rank 11: 17.99 + rank 12: 18.74 + rank 13: 20.12 + rank 14: 15.96 + rank 15: 16.24 + rank 16: 15.53 + rank 17: 15.67 + rank 18: 14.54 + rank 19: 14.84 + rank 20: 14.94 + rank 21: 15.38 + rank 22: 14.39 + rank 23: 13.88 + rank 24: 22.30 + rank 25: 21.51 + rank 26: 21.99 + rank 27: 22.25 + rank 28: 20.63 + rank 29: 18.30 + rank 30: 22.03 + rank 31: 22.99 + rank 32: 23.02 + rank 33: 23.17 + rank 34: 23.90 + rank 35: 25.16 + rank 36: 24.33 + rank 37: 25.13 + rank 38: 25.37 + rank 39: 26.07 + rank 40: 16.13 + rank 41: 15.76 + rank 42: 15.28 + rank 43: 15.36 + rank 44: 16.75 + rank 45: 15.49 + rank 46: 15.27 + rank 47: 15.54 + rank 48: 20.11 + rank 49: 18.86 + rank 50: 15.91 + rank 51: 21.93 + rank 52: 19.01 + rank 53: 18.99 + rank 54: 15.93 + rank 55: 19.08 + rank 56: 16.28 + rank 57: 20.34 + rank 58: 15.21 + rank 59: 15.54 + rank 60: 15.24 + rank 61: 15.59 + rank 62: 14.63 + rank 63: 15.14 + embedding-grads-all-reduce: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.03 + rank 3: 0.02 + rank 4: 0.03 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.03 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.03 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.04 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.03 + rank 60: 0.02 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.02 + all-grads-sync: + rank 0: 0.86 + rank 1: 0.77 + rank 2: 0.77 + rank 3: 0.84 + rank 4: 1.55 + rank 5: 0.79 + rank 6: 0.77 + rank 7: 0.77 + rank 8: 0.77 + rank 9: 0.96 + rank 10: 0.78 + rank 11: 0.79 + rank 12: 0.84 + rank 13: 0.85 + rank 14: 0.83 + rank 15: 0.85 + rank 16: 0.80 + rank 17: 0.91 + rank 18: 0.78 + rank 19: 0.77 + rank 20: 0.79 + rank 21: 0.77 + rank 22: 0.76 + rank 23: 0.81 + rank 24: 1.24 + rank 25: 1.14 + rank 26: 1.17 + rank 27: 1.68 + rank 28: 1.41 + rank 29: 1.27 + rank 30: 1.17 + rank 31: 1.28 + rank 32: 1.53 + rank 33: 1.30 + rank 34: 1.16 + rank 35: 1.37 + rank 36: 1.22 + rank 37: 1.15 + rank 38: 1.13 + rank 39: 1.13 + rank 40: 0.80 + rank 41: 0.79 + rank 42: 0.78 + rank 43: 1.11 + rank 44: 0.80 + rank 45: 0.92 + rank 46: 0.95 + rank 47: 0.79 + rank 48: 0.90 + rank 49: 0.89 + rank 50: 0.99 + rank 51: 1.65 + rank 52: 0.85 + rank 53: 1.17 + rank 54: 1.03 + rank 55: 1.13 + rank 56: 1.14 + rank 57: 0.77 + rank 58: 0.90 + rank 59: 1.34 + rank 60: 0.81 + rank 61: 0.81 + rank 62: 0.95 + rank 63: 0.81 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.05 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.02 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.02 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.09 + rank 1: 1.81 + rank 2: 1.79 + rank 3: 2.13 + rank 4: 1.94 + rank 5: 1.87 + rank 6: 1.88 + rank 7: 1.97 + rank 8: 1.93 + rank 9: 2.17 + rank 10: 1.85 + rank 11: 1.83 + rank 12: 1.93 + rank 13: 1.78 + rank 14: 1.98 + rank 15: 2.00 + rank 16: 2.45 + rank 17: 2.16 + rank 18: 2.20 + rank 19: 2.22 + rank 20: 2.27 + rank 21: 2.26 + rank 22: 2.54 + rank 23: 2.29 + rank 24: 1.96 + rank 25: 2.34 + rank 26: 2.05 + rank 27: 2.26 + rank 28: 2.39 + rank 29: 2.42 + rank 30: 2.18 + rank 31: 2.12 + rank 32: 2.06 + rank 33: 1.99 + rank 34: 2.10 + rank 35: 2.35 + rank 36: 2.43 + rank 37: 2.09 + rank 38: 2.31 + rank 39: 1.98 + rank 40: 1.75 + rank 41: 1.82 + rank 42: 1.81 + rank 43: 1.84 + rank 44: 1.90 + rank 45: 2.03 + rank 46: 1.72 + rank 47: 1.71 + rank 48: 2.06 + rank 49: 1.86 + rank 50: 2.43 + rank 51: 2.13 + rank 52: 1.97 + rank 53: 1.85 + rank 54: 1.86 + rank 55: 1.98 + rank 56: 1.96 + rank 57: 2.51 + rank 58: 1.82 + rank 59: 1.81 + rank 60: 1.80 + rank 61: 1.75 + rank 62: 1.92 + rank 63: 1.81 + optimizer-inner-step: + rank 0: 42.14 + rank 1: 42.07 + rank 2: 41.67 + rank 3: 41.90 + rank 4: 41.64 + rank 5: 41.77 + rank 6: 42.21 + rank 7: 42.07 + rank 8: 41.84 + rank 9: 41.88 + rank 10: 41.80 + rank 11: 41.83 + rank 12: 41.85 + rank 13: 42.09 + rank 14: 41.81 + rank 15: 41.71 + rank 16: 42.24 + rank 17: 41.92 + rank 18: 42.08 + rank 19: 41.88 + rank 20: 42.00 + rank 21: 41.83 + rank 22: 42.30 + rank 23: 41.98 + rank 24: 41.68 + rank 25: 41.86 + rank 26: 41.83 + rank 27: 41.96 + rank 28: 46.16 + rank 29: 41.72 + rank 30: 41.74 + rank 31: 42.07 + rank 32: 41.90 + rank 33: 41.75 + rank 34: 41.80 + rank 35: 50.24 + rank 36: 45.79 + rank 37: 41.89 + rank 38: 41.73 + rank 39: 41.73 + rank 40: 41.79 + rank 41: 41.89 + rank 42: 42.02 + rank 43: 42.10 + rank 44: 41.55 + rank 45: 42.28 + rank 46: 41.99 + rank 47: 41.61 + rank 48: 42.67 + rank 49: 41.81 + rank 50: 53.27 + rank 51: 41.57 + rank 52: 41.94 + rank 53: 41.86 + rank 54: 41.89 + rank 55: 41.82 + rank 56: 41.55 + rank 57: 47.33 + rank 58: 41.84 + rank 59: 41.71 + rank 60: 42.06 + rank 61: 41.61 + rank 62: 41.78 + rank 63: 41.63 + optimizer-copy-main-to-model-params: + rank 0: 4.44 + rank 1: 3.87 + rank 2: 3.81 + rank 3: 4.36 + rank 4: 3.87 + rank 5: 3.96 + rank 6: 4.01 + rank 7: 4.06 + rank 8: 3.92 + rank 9: 4.51 + rank 10: 3.88 + rank 11: 3.89 + rank 12: 4.02 + rank 13: 3.84 + rank 14: 4.09 + rank 15: 4.25 + rank 16: 4.46 + rank 17: 4.08 + rank 18: 4.14 + rank 19: 4.11 + rank 20: 4.11 + rank 21: 4.22 + rank 22: 4.69 + rank 23: 4.21 + rank 24: 3.99 + rank 25: 4.40 + rank 26: 4.05 + rank 27: 4.21 + rank 28: 4.54 + rank 29: 4.46 + rank 30: 4.43 + rank 31: 4.22 + rank 32: 4.21 + rank 33: 4.07 + rank 34: 4.16 + rank 35: 4.86 + rank 36: 4.51 + rank 37: 4.25 + rank 38: 4.73 + rank 39: 4.03 + rank 40: 3.82 + rank 41: 4.01 + rank 42: 4.02 + rank 43: 4.01 + rank 44: 4.07 + rank 45: 4.33 + rank 46: 3.81 + rank 47: 3.77 + rank 48: 4.31 + rank 49: 4.09 + rank 50: 5.36 + rank 51: 4.30 + rank 52: 4.32 + rank 53: 4.03 + rank 54: 3.98 + rank 55: 4.05 + rank 56: 3.91 + rank 57: 4.85 + rank 58: 3.91 + rank 59: 3.84 + rank 60: 3.89 + rank 61: 3.78 + rank 62: 4.00 + rank 63: 3.75 + optimizer: + rank 0: 53.81 + rank 1: 52.95 + rank 2: 52.44 + rank 3: 53.51 + rank 4: 52.44 + rank 5: 52.75 + rank 6: 53.27 + rank 7: 53.16 + rank 8: 52.77 + rank 9: 53.66 + rank 10: 52.67 + rank 11: 52.71 + rank 12: 52.88 + rank 13: 52.95 + rank 14: 52.99 + rank 15: 53.09 + rank 16: 54.22 + rank 17: 53.20 + rank 18: 53.45 + rank 19: 53.25 + rank 20: 53.33 + rank 21: 53.41 + rank 22: 54.51 + rank 23: 53.52 + rank 24: 52.65 + rank 25: 53.37 + rank 26: 52.88 + rank 27: 53.20 + rank 28: 57.88 + rank 29: 53.33 + rank 30: 53.35 + rank 31: 53.30 + rank 32: 53.11 + rank 33: 52.74 + rank 34: 52.92 + rank 35: 62.25 + rank 36: 57.39 + rank 37: 53.12 + rank 38: 53.59 + rank 39: 52.69 + rank 40: 52.57 + rank 41: 52.94 + rank 42: 53.06 + rank 43: 53.13 + rank 44: 52.68 + rank 45: 53.75 + rank 46: 52.72 + rank 47: 52.30 + rank 48: 54.03 + rank 49: 52.86 + rank 50: 66.04 + rank 51: 52.87 + rank 52: 53.34 + rank 53: 52.88 + rank 54: 52.85 + rank 55: 52.85 + rank 56: 52.43 + rank 57: 59.52 + rank 58: 52.76 + rank 59: 52.51 + rank 60: 52.92 + rank 61: 52.36 + rank 62: 52.82 + rank 63: 52.30 + interval-time: + rank 0: 0.68 + rank 1: 0.76 + rank 2: 0.78 + rank 3: 0.67 + rank 4: 0.82 + rank 5: 0.68 + rank 6: 0.67 + rank 7: 0.66 + rank 8: 0.76 + rank 9: 0.75 + rank 10: 0.72 + rank 11: 0.71 + rank 12: 0.69 + rank 13: 0.76 + rank 14: 0.71 + rank 15: 0.68 + rank 16: 0.70 + rank 17: 0.70 + rank 18: 0.68 + rank 19: 0.69 + rank 20: 0.69 + rank 21: 0.70 + rank 22: 0.68 + rank 23: 0.76 + rank 24: 0.92 + rank 25: 0.78 + rank 26: 0.83 + rank 27: 0.90 + rank 28: 0.87 + rank 29: 0.77 + rank 30: 0.77 + rank 31: 0.85 + rank 32: 0.87 + rank 33: 0.88 + rank 34: 0.75 + rank 35: 0.79 + rank 36: 0.88 + rank 37: 0.79 + rank 38: 0.87 + rank 39: 0.78 + rank 40: 0.69 + rank 41: 0.76 + rank 42: 0.78 + rank 43: 0.78 + rank 44: 0.77 + rank 45: 0.69 + rank 46: 0.78 + rank 47: 0.77 + rank 48: 0.78 + rank 49: 0.77 + rank 50: 0.76 + rank 51: 1.07 + rank 52: 0.94 + rank 53: 0.81 + rank 54: 0.80 + rank 55: 0.75 + rank 56: 0.68 + rank 57: 0.69 + rank 58: 0.68 + rank 59: 0.69 + rank 60: 0.67 + rank 61: 0.68 + rank 62: 0.69 + rank 63: 2.34 + [2026-10-11 03:11:25] iteration 6/ 2384 | consumed samples: 192 | elapsed time per iteration (ms): 147273.3 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.71% | FLOPs estimator: workload-aware | logical tokens/s: 113918.8 | valid tokens/s: 113918.8 | physical tokens/s: 114617.2 | loss tokens/s: 85171.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.508636E-01 | lm-global-supervised-tokens: 1.254354E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.722204E-02 | lm-token-buffer-mib: 1.164819E+02 | global_load_balancing_loss: 1.002776E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11235.927 | params rms: 0.060839 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:16:56.016663 | finish at 2026-10-15 04:28:21 + [2026-10-11 03:14:43] iteration 7/ 2384 | consumed samples: 224 | elapsed time per iteration (ms): 198166.3 | throughput per GPU (TFLOP/s/GPU): 28.2 | MFU 9.04% | FLOPs estimator: workload-aware | logical tokens/s: 84662.2 | valid tokens/s: 84662.2 | physical tokens/s: 85119.1 | loss tokens/s: 64988.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.360238E-01 | lm-global-supervised-tokens: 1.287843E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.055219E-02 | lm-token-buffer-mib: 1.161914E+02 | global_load_balancing_loss: 1.001790E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11235.874 | params rms: 0.060839 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:50:41.274837 | finish at 2026-10-16 14:05:24 + [2026-10-11 03:17:28] iteration 8/ 2384 | consumed samples: 256 | elapsed time per iteration (ms): 165472.8 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.94% | FLOPs estimator: workload-aware | logical tokens/s: 101389.3 | valid tokens/s: 101389.3 | physical tokens/s: 101861.3 | loss tokens/s: 77607.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.457377E-01 | lm-global-supervised-tokens: 1.284196E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.053438E-02 | lm-token-buffer-mib: 1.162309E+02 | global_load_balancing_loss: 1.000629E+00 | loss scale: 1.0 | grad norm: 0.382 | params norm: 11235.822 | params rms: 0.060838 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:12:43.308260 | finish at 2026-10-15 16:30:12 + [2026-10-11 03:20:43] iteration 9/ 2384 | consumed samples: 288 | elapsed time per iteration (ms): 194717.5 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.98% | FLOPs estimator: workload-aware | logical tokens/s: 86161.6 | valid tokens/s: 86161.6 | physical tokens/s: 86600.5 | loss tokens/s: 66188.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.578102E-01 | lm-global-supervised-tokens: 1.288805E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.905114E-02 | lm-token-buffer-mib: 1.166614E+02 | global_load_balancing_loss: 9.988152E-01 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11235.773 | params rms: 0.060838 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:27:34.048842 | finish at 2026-10-16 11:48:17 + [2026-10-11 03:23:31] iteration 10/ 2384 | consumed samples: 320 | elapsed time per iteration (ms): 168217.9 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.20% | FLOPs estimator: workload-aware | logical tokens/s: 99734.9 | valid tokens/s: 99734.9 | physical tokens/s: 100493.5 | loss tokens/s: 81593.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.999952E-01 | lm-global-supervised-tokens: 1.372542E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.198694E-02 | lm-token-buffer-mib: 1.174754E+02 | global_load_balancing_loss: 9.995948E-01 | loss scale: 1.0 | grad norm: 0.455 | params norm: 11235.726 | params rms: 0.060838 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:55:49.214883 | finish at 2026-10-15 18:19:21 + [2026-10-11 03:26:06] iteration 11/ 2384 | consumed samples: 352 | elapsed time per iteration (ms): 154272.3 | throughput per GPU (TFLOP/s/GPU): 31.5 | MFU 10.09% | FLOPs estimator: workload-aware | logical tokens/s: 108750.3 | valid tokens/s: 108750.3 | physical tokens/s: 109358.0 | loss tokens/s: 84301.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.979868E-01 | lm-global-supervised-tokens: 1.300542E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.665858E-02 | lm-token-buffer-mib: 1.168755E+02 | global_load_balancing_loss: 9.985627E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11235.677 | params rms: 0.060838 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:41:28.246553 | finish at 2026-10-15 09:07:34 + [2026-10-11 03:28:27] iteration 12/ 2384 | consumed samples: 384 | elapsed time per iteration (ms): 141863.9 | throughput per GPU (TFLOP/s/GPU): 30.1 | MFU 9.64% | FLOPs estimator: workload-aware | logical tokens/s: 118262.4 | valid tokens/s: 118262.4 | physical tokens/s: 119153.3 | loss tokens/s: 91610.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.139291E-01 | lm-global-supervised-tokens: 1.299624E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.735379E-02 | lm-token-buffer-mib: 1.165738E+02 | global_load_balancing_loss: 9.992263E-01 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11235.626 | params rms: 0.060837 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:28:21.116947 | finish at 2026-10-15 00:56:49 + [2026-10-11 03:30:51] iteration 13/ 2384 | consumed samples: 416 | elapsed time per iteration (ms): 143487.7 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.57% | FLOPs estimator: workload-aware | logical tokens/s: 116924.2 | valid tokens/s: 116924.2 | physical tokens/s: 117279.9 | loss tokens/s: 87508.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.235981E-01 | lm-global-supervised-tokens: 1.255644E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.710486E-02 | lm-token-buffer-mib: 1.160972E+02 | global_load_balancing_loss: 1.002895E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11235.573 | params rms: 0.060837 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:30:09.325925 | finish at 2026-10-15 02:01:00 + [2026-10-11 03:34:12] iteration 14/ 2384 | consumed samples: 448 | elapsed time per iteration (ms): 201331.8 | throughput per GPU (TFLOP/s/GPU): 26.8 | MFU 8.58% | FLOPs estimator: workload-aware | logical tokens/s: 83331.0 | valid tokens/s: 83331.0 | physical tokens/s: 83979.8 | loss tokens/s: 65402.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.443823E-01 | lm-global-supervised-tokens: 1.316755E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.682729E-02 | lm-token-buffer-mib: 1.160890E+02 | global_load_balancing_loss: 1.005247E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11235.517 | params rms: 0.060837 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:32:36.364832 | finish at 2026-10-16 16:06:49 + [2026-10-11 03:36:34] iteration 15/ 2384 | consumed samples: 480 | elapsed time per iteration (ms): 141833.2 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.68% | FLOPs estimator: workload-aware | logical tokens/s: 118288.3 | valid tokens/s: 118288.3 | physical tokens/s: 118995.2 | loss tokens/s: 92312.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.521369E-01 | lm-global-supervised-tokens: 1.309292E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.687739E-02 | lm-token-buffer-mib: 1.172271E+02 | global_load_balancing_loss: 1.004266E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11235.457 | params rms: 0.060836 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:20:02.769979 | finish at 2026-10-15 00:56:37 + [2026-10-11 03:38:58] iteration 16/ 2384 | consumed samples: 512 | elapsed time per iteration (ms): 144267.5 | throughput per GPU (TFLOP/s/GPU): 32.6 | MFU 10.46% | FLOPs estimator: workload-aware | logical tokens/s: 116292.1 | valid tokens/s: 116292.1 | physical tokens/s: 116555.5 | loss tokens/s: 82159.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.221037E-01 | lm-global-supervised-tokens: 1.185287E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.087330E-02 | lm-token-buffer-mib: 1.163174E+02 | global_load_balancing_loss: 9.987578E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11235.394 | params rms: 0.060836 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:53:45.466919 | finish at 2026-10-15 02:32:44 + [2026-10-11 03:41:21] iteration 17/ 2384 | consumed samples: 544 | elapsed time per iteration (ms): 142329.3 | throughput per GPU (TFLOP/s/GPU): 35.2 | MFU 11.28% | FLOPs estimator: workload-aware | logical tokens/s: 117875.7 | valid tokens/s: 117875.7 | physical tokens/s: 118374.1 | loss tokens/s: 90216.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.377000E-01 | lm-global-supervised-tokens: 1.284042E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.009236E-02 | lm-token-buffer-mib: 1.171209E+02 | global_load_balancing_loss: 1.001293E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11235.330 | params rms: 0.060836 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:34:53.427531 | finish at 2026-10-15 01:16:14 + [2026-10-11 03:43:41] iteration 18/ 2384 | consumed samples: 576 | elapsed time per iteration (ms): 140498.3 | throughput per GPU (TFLOP/s/GPU): 28.5 | MFU 9.13% | FLOPs estimator: workload-aware | logical tokens/s: 119412.2 | valid tokens/s: 119412.2 | physical tokens/s: 119967.9 | loss tokens/s: 93205.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.040262E-01 | lm-global-supervised-tokens: 1.309522E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.057930E-02 | lm-token-buffer-mib: 1.171599E+02 | global_load_balancing_loss: 1.000049E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11235.266 | params rms: 0.060835 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:20:19.074440 | finish at 2026-10-15 00:04:00 + [2026-10-11 03:46:07] iteration 19/ 2384 | consumed samples: 608 | elapsed time per iteration (ms): 145992.4 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.57% | FLOPs estimator: workload-aware | logical tokens/s: 114918.2 | valid tokens/s: 114918.2 | physical tokens/s: 115535.8 | loss tokens/s: 90249.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.355386E-01 | lm-global-supervised-tokens: 1.317576E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.984615E-02 | lm-token-buffer-mib: 1.168411E+02 | global_load_balancing_loss: 9.996197E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11235.200 | params rms: 0.060835 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:54:32.130715 | finish at 2026-10-15 03:40:39 + [2026-10-11 03:48:41] iteration 20/ 2384 | consumed samples: 640 | elapsed time per iteration (ms): 153937.2 | throughput per GPU (TFLOP/s/GPU): 30.8 | MFU 9.88% | FLOPs estimator: workload-aware | logical tokens/s: 108987.1 | valid tokens/s: 108987.1 | physical tokens/s: 109619.7 | loss tokens/s: 85054.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.578763E-01 | lm-global-supervised-tokens: 1.309309E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.572997E-02 | lm-token-buffer-mib: 1.165820E+02 | global_load_balancing_loss: 9.976768E-01 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11235.134 | params rms: 0.060835 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:05:07.620998 | finish at 2026-10-15 08:53:49 + [2026-10-11 03:51:04] iteration 21/ 2384 | consumed samples: 672 | elapsed time per iteration (ms): 142524.2 | throughput per GPU (TFLOP/s/GPU): 32.7 | MFU 10.47% | FLOPs estimator: workload-aware | logical tokens/s: 117714.8 | valid tokens/s: 117714.8 | physical tokens/s: 118329.4 | loss tokens/s: 86986.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.763938E-01 | lm-global-supervised-tokens: 1.239772E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.119132E-02 | lm-token-buffer-mib: 1.171418E+02 | global_load_balancing_loss: 1.000381E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11235.068 | params rms: 0.060834 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:33:04.683948 | finish at 2026-10-15 01:24:08 + [2026-10-11 03:53:26] iteration 22/ 2384 | consumed samples: 704 | elapsed time per iteration (ms): 142086.5 | throughput per GPU (TFLOP/s/GPU): 35.4 | MFU 11.36% | FLOPs estimator: workload-aware | logical tokens/s: 118077.4 | valid tokens/s: 118077.4 | physical tokens/s: 118492.1 | loss tokens/s: 93048.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.652446E-01 | lm-global-supervised-tokens: 1.322093E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.004881E-02 | lm-token-buffer-mib: 1.162617E+02 | global_load_balancing_loss: 9.995368E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11235.003 | params rms: 0.060834 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:13:28.355069 | finish at 2026-10-15 01:06:54 + [2026-10-11 03:55:43] iteration 23/ 2384 | consumed samples: 736 | elapsed time per iteration (ms): 136847.2 | throughput per GPU (TFLOP/s/GPU): 31.2 | MFU 9.99% | FLOPs estimator: workload-aware | logical tokens/s: 122598.0 | valid tokens/s: 122598.0 | physical tokens/s: 123204.7 | loss tokens/s: 98898.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.004248E-01 | lm-global-supervised-tokens: 1.353398E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.771559E-02 | lm-token-buffer-mib: 1.165356E+02 | global_load_balancing_loss: 1.003201E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11234.934 | params rms: 0.060834 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:44:56.275593 | finish at 2026-10-14 21:40:39 + [2026-10-11 03:58:07] iteration 24/ 2384 | consumed samples: 768 | elapsed time per iteration (ms): 144525.1 | throughput per GPU (TFLOP/s/GPU): 39.5 | MFU 12.66% | FLOPs estimator: workload-aware | logical tokens/s: 116084.9 | valid tokens/s: 116084.9 | physical tokens/s: 116499.2 | loss tokens/s: 83107.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.112203E-01 | lm-global-supervised-tokens: 1.201110E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.701000E-02 | lm-token-buffer-mib: 1.161130E+02 | global_load_balancing_loss: 1.001894E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11234.864 | params rms: 0.060833 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:44:39.160585 | finish at 2026-10-15 02:42:46 + [2026-10-11 04:00:37] iteration 25/ 2384 | consumed samples: 800 | elapsed time per iteration (ms): 149518.8 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.88% | FLOPs estimator: workload-aware | logical tokens/s: 112208.0 | valid tokens/s: 112207.9 | physical tokens/s: 112628.2 | loss tokens/s: 84575.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.863483E-01 | lm-global-supervised-tokens: 1.264563E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.001394E-02 | lm-token-buffer-mib: 1.169476E+02 | global_load_balancing_loss: 1.001720E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11234.786 | params rms: 0.060833 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:58:34.846998 | finish at 2026-10-15 05:59:11 + [2026-10-11 04:03:14] iteration 26/ 2384 | consumed samples: 832 | elapsed time per iteration (ms): 157494.5 | throughput per GPU (TFLOP/s/GPU): 32.9 | MFU 10.54% | FLOPs estimator: workload-aware | logical tokens/s: 106525.5 | valid tokens/s: 106525.5 | physical tokens/s: 106963.2 | loss tokens/s: 80543.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.151273E-01 | lm-global-supervised-tokens: 1.268519E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.599864E-02 | lm-token-buffer-mib: 1.163286E+02 | global_load_balancing_loss: 1.001455E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11234.710 | params rms: 0.060832 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:09:31.945363 | finish at 2026-10-15 11:12:46 + [2026-10-11 04:05:41] iteration 27/ 2384 | consumed samples: 864 | elapsed time per iteration (ms): 146857.8 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.72% | FLOPs estimator: workload-aware | logical tokens/s: 114241.1 | valid tokens/s: 114241.1 | physical tokens/s: 114797.5 | loss tokens/s: 90822.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.172650E-01 | lm-global-supervised-tokens: 1.333799E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.165148E-02 | lm-token-buffer-mib: 1.163824E+02 | global_load_balancing_loss: 1.000251E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11234.636 | params rms: 0.060832 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:09:03.920595 | finish at 2026-10-15 04:14:45 + [2026-10-11 04:08:21] iteration 28/ 2384 | consumed samples: 896 | elapsed time per iteration (ms): 159798.6 | throughput per GPU (TFLOP/s/GPU): 36.0 | MFU 11.53% | FLOPs estimator: workload-aware | logical tokens/s: 104989.6 | valid tokens/s: 104989.6 | physical tokens/s: 105322.8 | loss tokens/s: 79573.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.734692E-01 | lm-global-supervised-tokens: 1.271565E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.693356E-02 | lm-token-buffer-mib: 1.163675E+02 | global_load_balancing_loss: 1.001725E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11234.562 | params rms: 0.060831 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:34:45.467237 | finish at 2026-10-15 12:43:06 + [2026-10-11 04:10:48] iteration 29/ 2384 | consumed samples: 928 | elapsed time per iteration (ms): 147472.1 | throughput per GPU (TFLOP/s/GPU): 37.8 | MFU 12.12% | FLOPs estimator: workload-aware | logical tokens/s: 113764.9 | valid tokens/s: 113764.9 | physical tokens/s: 114055.9 | loss tokens/s: 88423.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.226446E-01 | lm-global-supervised-tokens: 1.304001E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.993469E-02 | lm-token-buffer-mib: 1.161298E+02 | global_load_balancing_loss: 9.999561E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11234.488 | params rms: 0.060831 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:28:16.882575 | finish at 2026-10-15 04:39:05 + [2026-10-11 04:13:32] iteration 30/ 2384 | consumed samples: 960 | elapsed time per iteration (ms): 164165.6 | throughput per GPU (TFLOP/s/GPU): 29.1 | MFU 9.32% | FLOPs estimator: workload-aware | logical tokens/s: 102196.7 | valid tokens/s: 102196.7 | physical tokens/s: 102560.0 | loss tokens/s: 81261.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.679047E-01 | lm-global-supervised-tokens: 1.334026E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.799077E-02 | lm-token-buffer-mib: 1.164015E+02 | global_load_balancing_loss: 9.993202E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11234.418 | params rms: 0.060831 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:20:45.933670 | finish at 2026-10-15 15:34:18 + [2026-10-11 04:16:31] iteration 31/ 2384 | consumed samples: 992 | elapsed time per iteration (ms): 178926.4 | throughput per GPU (TFLOP/s/GPU): 28.0 | MFU 8.99% | FLOPs estimator: workload-aware | logical tokens/s: 93765.7 | valid tokens/s: 93765.7 | physical tokens/s: 94419.7 | loss tokens/s: 68633.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.841778E-01 | lm-global-supervised-tokens: 1.228029E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.057551E-02 | lm-token-buffer-mib: 1.164425E+02 | global_load_balancing_loss: 9.989265E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11234.346 | params rms: 0.060830 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:56:53.722021 | finish at 2026-10-16 01:13:25 + [2026-10-11 04:19:00] iteration 32/ 2384 | consumed samples: 1024 | elapsed time per iteration (ms): 149093.9 | throughput per GPU (TFLOP/s/GPU): 28.9 | MFU 9.27% | FLOPs estimator: workload-aware | logical tokens/s: 112527.8 | valid tokens/s: 112527.8 | physical tokens/s: 113570.1 | loss tokens/s: 86338.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.083712E-01 | lm-global-supervised-tokens: 1.287257E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.053211E-02 | lm-token-buffer-mib: 1.166339E+02 | global_load_balancing_loss: 1.001207E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11234.273 | params rms: 0.060830 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:24:28.829166 | finish at 2026-10-15 05:43:29 + [2026-10-11 04:22:24] iteration 33/ 2384 | consumed samples: 1056 | elapsed time per iteration (ms): 203271.5 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 82536.0 | valid tokens/s: 82536.0 | physical tokens/s: 83262.8 | loss tokens/s: 64257.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.643971E-01 | lm-global-supervised-tokens: 1.306176E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.450064E-02 | lm-token-buffer-mib: 1.188302E+02 | global_load_balancing_loss: 1.000529E+00 | loss scale: 1.0 | grad norm: 0.259 | params norm: 11234.205 | params rms: 0.060830 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:44:51.326468 | finish at 2026-10-16 17:07:15 + [2026-10-11 04:24:59] iteration 34/ 2384 | consumed samples: 1088 | elapsed time per iteration (ms): 154859.9 | throughput per GPU (TFLOP/s/GPU): 35.3 | MFU 11.30% | FLOPs estimator: workload-aware | logical tokens/s: 108337.8 | valid tokens/s: 108337.8 | physical tokens/s: 108466.3 | loss tokens/s: 82569.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.598691E-01 | lm-global-supervised-tokens: 1.278668E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.949679E-02 | lm-token-buffer-mib: 1.160636E+02 | global_load_balancing_loss: 1.001790E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11234.138 | params rms: 0.060829 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:05:20.779002 | finish at 2026-10-15 09:30:19 + [2026-10-11 04:27:30] iteration 35/ 2384 | consumed samples: 1120 | elapsed time per iteration (ms): 151736.2 | throughput per GPU (TFLOP/s/GPU): 32.7 | MFU 10.48% | FLOPs estimator: workload-aware | logical tokens/s: 110568.1 | valid tokens/s: 110568.1 | physical tokens/s: 111088.6 | loss tokens/s: 87592.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.375346E-01 | lm-global-supervised-tokens: 1.329092E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.969377E-02 | lm-token-buffer-mib: 1.168348E+02 | global_load_balancing_loss: 1.003980E+00 | loss scale: 1.0 | grad norm: 0.772 | params norm: 11234.068 | params rms: 0.060829 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:00:28.389466 | finish at 2026-10-15 07:27:59 + [2026-10-11 04:29:55] iteration 36/ 2384 | consumed samples: 1152 | elapsed time per iteration (ms): 144266.7 | throughput per GPU (TFLOP/s/GPU): 31.2 | MFU 10.01% | FLOPs estimator: workload-aware | logical tokens/s: 116292.8 | valid tokens/s: 116292.8 | physical tokens/s: 116684.5 | loss tokens/s: 90999.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.402408E-01 | lm-global-supervised-tokens: 1.312814E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.609766E-02 | lm-token-buffer-mib: 1.162647E+02 | global_load_balancing_loss: 1.000644E+00 | loss scale: 1.0 | grad norm: 0.232 | params norm: 11233.993 | params rms: 0.060828 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:05:38.265970 | finish at 2026-10-15 02:35:33 + [2026-10-11 04:32:27] iteration 37/ 2384 | consumed samples: 1184 | elapsed time per iteration (ms): 151965.6 | throughput per GPU (TFLOP/s/GPU): 32.3 | MFU 10.36% | FLOPs estimator: workload-aware | logical tokens/s: 110401.2 | valid tokens/s: 110401.2 | physical tokens/s: 111191.9 | loss tokens/s: 86337.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.385550E-01 | lm-global-supervised-tokens: 1.312039E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.791569E-02 | lm-token-buffer-mib: 1.161842E+02 | global_load_balancing_loss: 1.000236E+00 | loss scale: 1.0 | grad norm: 0.897 | params norm: 11233.920 | params rms: 0.060828 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:04:23.234135 | finish at 2026-10-15 07:36:50 + [2026-10-11 04:34:50] iteration 38/ 2384 | consumed samples: 1216 | elapsed time per iteration (ms): 143195.3 | throughput per GPU (TFLOP/s/GPU): 32.1 | MFU 10.30% | FLOPs estimator: workload-aware | logical tokens/s: 117163.2 | valid tokens/s: 117163.2 | physical tokens/s: 117771.0 | loss tokens/s: 86482.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.386329E-01 | lm-global-supervised-tokens: 1.238391E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.780219E-02 | lm-token-buffer-mib: 1.161834E+02 | global_load_balancing_loss: 1.001121E+00 | loss scale: 1.0 | grad norm: 0.287 | params norm: 11233.844 | params rms: 0.060828 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:18:56.160057 | finish at 2026-10-15 01:53:46 + [2026-10-11 04:37:19] iteration 39/ 2384 | consumed samples: 1248 | elapsed time per iteration (ms): 149504.4 | throughput per GPU (TFLOP/s/GPU): 32.2 | MFU 10.32% | FLOPs estimator: workload-aware | logical tokens/s: 112218.9 | valid tokens/s: 112218.9 | physical tokens/s: 112469.7 | loss tokens/s: 90158.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.923172E-01 | lm-global-supervised-tokens: 1.347908E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.027857E-02 | lm-token-buffer-mib: 1.163637E+02 | global_load_balancing_loss: 1.000150E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11233.771 | params rms: 0.060827 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:23:07.802380 | finish at 2026-10-15 06:00:27 + [2026-10-11 04:40:11] iteration 40/ 2384 | consumed samples: 1280 | elapsed time per iteration (ms): 172104.9 | throughput per GPU (TFLOP/s/GPU): 28.8 | MFU 9.23% | FLOPs estimator: workload-aware | logical tokens/s: 97482.3 | valid tokens/s: 97482.3 | physical tokens/s: 97709.5 | loss tokens/s: 73683.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.564656E-01 | lm-global-supervised-tokens: 1.268123E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.910139E-02 | lm-token-buffer-mib: 1.160546E+02 | global_load_balancing_loss: 1.000468E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11233.698 | params rms: 0.060827 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:03:33.948477 | finish at 2026-10-15 20:43:45 +W1011 04:40:46.736000 622559 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 04:40:46.736000 622559 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +times across ranks (ms): + save-checkpoint: + rank 0: 111165.17 + rank 1: 111165.15 + rank 2: 111165.11 + rank 3: 111165.11 + rank 4: 111165.12 + rank 5: 111165.20 + rank 6: 111165.20 + rank 7: 111165.13 + rank 8: 111163.62 + rank 9: 111163.66 + rank 10: 111163.62 + rank 11: 111163.58 + rank 12: 111163.61 + rank 13: 111163.62 + rank 14: 111163.62 + rank 15: 111163.62 + rank 16: 111162.23 + rank 17: 111162.30 + rank 18: 111162.33 + rank 19: 111162.30 + rank 20: 111162.54 + rank 21: 111162.25 + rank 22: 111162.27 + rank 23: 111162.28 + rank 24: 111164.48 + rank 25: 111164.62 + rank 26: 111164.43 + rank 27: 111164.45 + rank 28: 111164.58 + rank 29: 111164.43 + rank 30: 111164.47 + rank 31: 111164.45 + rank 32: 111163.01 + rank 33: 111162.98 + rank 34: 111163.02 + rank 35: 111163.02 + rank 36: 111163.06 + rank 37: 111162.98 + rank 38: 111162.98 + rank 39: 111163.03 + rank 40: 111163.09 + rank 41: 111163.11 + rank 42: 111163.12 + rank 43: 111163.11 + rank 44: 111163.04 + rank 45: 111163.12 + rank 46: 111163.12 + rank 47: 111163.05 + rank 48: 111170.47 + rank 49: 111170.39 + rank 50: 111170.43 + rank 51: 111170.41 + rank 52: 111170.38 + rank 53: 111170.40 + rank 54: 111170.38 + rank 55: 111170.42 + rank 56: 111162.80 + rank 57: 111162.80 + rank 58: 111162.80 + rank 59: 111162.79 + rank 60: 111162.80 + rank 61: 111162.80 + rank 62: 111162.78 + rank 63: 111162.80 + [2026-10-11 04:44:19] iteration 41/ 2384 | consumed samples: 1312 | elapsed time per iteration (ms): 136237.8 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 123146.3 | valid tokens/s: 123146.3 | physical tokens/s: 123621.2 | loss tokens/s: 98154.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.084753E-01 | lm-global-supervised-tokens: 1.337232E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.309955E-01 | lm-token-buffer-mib: 1.162590E+02 | global_load_balancing_loss: 1.000393E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11233.623 | params rms: 0.060826 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:40:05.262883 | finish at 2026-10-14 21:24:24 + [2026-10-11 04:46:46] iteration 42/ 2384 | consumed samples: 1344 | elapsed time per iteration (ms): 147681.6 | throughput per GPU (TFLOP/s/GPU): 32.0 | MFU 10.25% | FLOPs estimator: workload-aware | logical tokens/s: 113603.9 | valid tokens/s: 113603.9 | physical tokens/s: 114269.6 | loss tokens/s: 83357.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.387081E-01 | lm-global-supervised-tokens: 1.231040E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.320354E-02 | lm-token-buffer-mib: 1.160887E+02 | global_load_balancing_loss: 9.996276E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11233.550 | params rms: 0.060826 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:04:30.295694 | finish at 2026-10-15 04:51:17 + [2026-10-11 04:49:09] iteration 43/ 2384 | consumed samples: 1376 | elapsed time per iteration (ms): 142894.3 | throughput per GPU (TFLOP/s/GPU): 32.9 | MFU 10.56% | FLOPs estimator: workload-aware | logical tokens/s: 117409.9 | valid tokens/s: 117409.9 | physical tokens/s: 117972.1 | loss tokens/s: 88288.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.593851E-01 | lm-global-supervised-tokens: 1.261598E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.807454E-02 | lm-token-buffer-mib: 1.161546E+02 | global_load_balancing_loss: 1.001344E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11233.480 | params rms: 0.060826 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:55:15.517193 | finish at 2026-10-15 01:44:25 + [2026-10-11 04:51:36] iteration 44/ 2384 | consumed samples: 1408 | elapsed time per iteration (ms): 146991.0 | throughput per GPU (TFLOP/s/GPU): 36.1 | MFU 11.58% | FLOPs estimator: workload-aware | logical tokens/s: 114137.3 | valid tokens/s: 114137.3 | physical tokens/s: 114676.7 | loss tokens/s: 88062.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.724657E-01 | lm-global-supervised-tokens: 1.294441E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.730182E-02 | lm-token-buffer-mib: 1.163379E+02 | global_load_balancing_loss: 1.000035E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11233.410 | params rms: 0.060825 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:32:39.000664 | finish at 2026-10-15 04:24:15 + [2026-10-11 04:54:08] iteration 45/ 2384 | consumed samples: 1440 | elapsed time per iteration (ms): 151561.0 | throughput per GPU (TFLOP/s/GPU): 32.2 | MFU 10.33% | FLOPs estimator: workload-aware | logical tokens/s: 110696.0 | valid tokens/s: 110696.0 | physical tokens/s: 111384.9 | loss tokens/s: 81642.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.191113E-01 | lm-global-supervised-tokens: 1.237381E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.794292E-02 | lm-token-buffer-mib: 1.165800E+02 | global_load_balancing_loss: 9.988060E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11233.341 | params rms: 0.060825 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:28:21.125719 | finish at 2026-10-15 07:22:29 + [2026-10-11 04:56:42] iteration 46/ 2384 | consumed samples: 1472 | elapsed time per iteration (ms): 154301.3 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.05% | FLOPs estimator: workload-aware | logical tokens/s: 108730.2 | valid tokens/s: 108730.1 | physical tokens/s: 109300.5 | loss tokens/s: 85940.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.875088E-01 | lm-global-supervised-tokens: 1.326078E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.454705E-02 | lm-token-buffer-mib: 1.163910E+02 | global_load_balancing_loss: 1.001626E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11233.270 | params rms: 0.060825 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:12:36.515324 | finish at 2026-10-15 09:09:19 + [2026-10-11 04:59:12] iteration 47/ 2384 | consumed samples: 1504 | elapsed time per iteration (ms): 149592.1 | throughput per GPU (TFLOP/s/GPU): 36.8 | MFU 11.81% | FLOPs estimator: workload-aware | logical tokens/s: 112152.8 | valid tokens/s: 112152.8 | physical tokens/s: 112632.2 | loss tokens/s: 86683.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.066953E-01 | lm-global-supervised-tokens: 1.296711E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.244628E-02 | lm-token-buffer-mib: 1.168686E+02 | global_load_balancing_loss: 1.003161E+00 | loss scale: 1.0 | grad norm: 0.274 | params norm: 11233.196 | params rms: 0.060824 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:06:36.852258 | finish at 2026-10-15 06:05:49 + [2026-10-11 05:01:40] iteration 48/ 2384 | consumed samples: 1536 | elapsed time per iteration (ms): 148340.6 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.15% | FLOPs estimator: workload-aware | logical tokens/s: 113098.9 | valid tokens/s: 113098.9 | physical tokens/s: 113433.2 | loss tokens/s: 79863.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.121051E-01 | lm-global-supervised-tokens: 1.184704E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.891140E-02 | lm-token-buffer-mib: 1.165674E+02 | global_load_balancing_loss: 1.001289E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11233.110 | params rms: 0.060824 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:15:23.711807 | finish at 2026-10-15 05:17:04 + [2026-10-11 05:04:10] iteration 49/ 2384 | consumed samples: 1568 | elapsed time per iteration (ms): 149857.7 | throughput per GPU (TFLOP/s/GPU): 35.7 | MFU 11.44% | FLOPs estimator: workload-aware | logical tokens/s: 111953.9 | valid tokens/s: 111953.9 | physical tokens/s: 112246.0 | loss tokens/s: 80039.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.219463E-01 | lm-global-supervised-tokens: 1.199448E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.033857E-02 | lm-token-buffer-mib: 1.164464E+02 | global_load_balancing_loss: 1.002003E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11233.037 | params rms: 0.060823 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:11:57.804911 | finish at 2026-10-15 06:16:08 + [2026-10-11 05:06:50] iteration 50/ 2384 | consumed samples: 1600 | elapsed time per iteration (ms): 159871.9 | throughput per GPU (TFLOP/s/GPU): 26.6 | MFU 8.52% | FLOPs estimator: workload-aware | logical tokens/s: 104941.6 | valid tokens/s: 104941.6 | physical tokens/s: 105838.4 | loss tokens/s: 82380.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.357934E-01 | lm-global-supervised-tokens: 1.317033E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.082613E-02 | lm-token-buffer-mib: 1.170628E+02 | global_load_balancing_loss: 9.998657E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11232.971 | params rms: 0.060823 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:39:00.944119 | finish at 2026-10-15 12:45:51 + [2026-10-11 05:09:14] iteration 51/ 2384 | consumed samples: 1632 | elapsed time per iteration (ms): 144405.3 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 116181.1 | valid tokens/s: 116181.1 | physical tokens/s: 116540.4 | loss tokens/s: 88043.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.365939E-01 | lm-global-supervised-tokens: 1.271391E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.067546E-02 | lm-token-buffer-mib: 1.166176E+02 | global_load_balancing_loss: 1.001943E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11232.911 | params rms: 0.060823 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:34:57.618069 | finish at 2026-10-15 02:44:12 + [2026-10-11 05:11:38] iteration 52/ 2384 | consumed samples: 1664 | elapsed time per iteration (ms): 143507.9 | throughput per GPU (TFLOP/s/GPU): 32.6 | MFU 10.44% | FLOPs estimator: workload-aware | logical tokens/s: 116907.8 | valid tokens/s: 116907.8 | physical tokens/s: 117312.2 | loss tokens/s: 88730.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.526121E-01 | lm-global-supervised-tokens: 1.273346E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.797454E-02 | lm-token-buffer-mib: 1.167576E+02 | global_load_balancing_loss: 1.000092E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11232.854 | params rms: 0.060822 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:57:40.531218 | finish at 2026-10-15 02:09:18 + [2026-10-11 05:13:56] iteration 53/ 2384 | consumed samples: 1696 | elapsed time per iteration (ms): 137888.4 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.05% | FLOPs estimator: workload-aware | logical tokens/s: 121672.2 | valid tokens/s: 121672.2 | physical tokens/s: 121987.1 | loss tokens/s: 88991.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.630099E-01 | lm-global-supervised-tokens: 1.227088E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.772757E-02 | lm-token-buffer-mib: 1.161328E+02 | global_load_balancing_loss: 9.977601E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11232.802 | params rms: 0.060822 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:16:57.857803 | finish at 2026-10-14 22:30:54 + [2026-10-11 05:17:34] iteration 54/ 2384 | consumed samples: 1728 | elapsed time per iteration (ms): 217901.8 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 76994.4 | valid tokens/s: 76994.4 | physical tokens/s: 77922.7 | loss tokens/s: 58391.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.936681E-01 | lm-global-supervised-tokens: 1.272366E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.757513E-02 | lm-token-buffer-mib: 1.164690E+02 | global_load_balancing_loss: 9.988199E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11232.753 | params rms: 0.060822 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:01:51.136034 | finish at 2026-10-17 02:19:25 + [2026-10-11 05:20:03] iteration 55/ 2384 | consumed samples: 1760 | elapsed time per iteration (ms): 149762.8 | throughput per GPU (TFLOP/s/GPU): 36.6 | MFU 11.72% | FLOPs estimator: workload-aware | logical tokens/s: 112025.2 | valid tokens/s: 112025.2 | physical tokens/s: 112559.0 | loss tokens/s: 81684.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.228813E-01 | lm-global-supervised-tokens: 1.223336E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.624684E-02 | lm-token-buffer-mib: 1.168952E+02 | global_load_balancing_loss: 9.998854E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11232.705 | params rms: 0.060821 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:53:17.560039 | finish at 2026-10-15 06:13:21 + [2026-10-11 05:22:30] iteration 56/ 2384 | consumed samples: 1792 | elapsed time per iteration (ms): 146879.9 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.94% | FLOPs estimator: workload-aware | logical tokens/s: 114223.8 | valid tokens/s: 114223.8 | physical tokens/s: 114446.9 | loss tokens/s: 88408.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.519377E-01 | lm-global-supervised-tokens: 1.298536E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.888138E-02 | lm-token-buffer-mib: 1.161377E+02 | global_load_balancing_loss: 9.991996E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11232.657 | params rms: 0.060821 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:58:56.443783 | finish at 2026-10-15 04:21:27 + [2026-10-11 05:24:55] iteration 57/ 2384 | consumed samples: 1824 | elapsed time per iteration (ms): 144518.0 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.77% | FLOPs estimator: workload-aware | logical tokens/s: 116090.6 | valid tokens/s: 116090.6 | physical tokens/s: 116644.4 | loss tokens/s: 87759.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.490143E-01 | lm-global-supervised-tokens: 1.268285E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.941716E-02 | lm-token-buffer-mib: 1.163905E+02 | global_load_balancing_loss: 1.001742E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11232.607 | params rms: 0.060821 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:24:53.376861 | finish at 2026-10-15 02:49:48 + [2026-10-11 05:27:16] iteration 58/ 2384 | consumed samples: 1856 | elapsed time per iteration (ms): 141428.2 | throughput per GPU (TFLOP/s/GPU): 35.0 | MFU 11.22% | FLOPs estimator: workload-aware | logical tokens/s: 118627.1 | valid tokens/s: 118627.0 | physical tokens/s: 119082.6 | loss tokens/s: 91508.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.929608E-01 | lm-global-supervised-tokens: 1.294188E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.957588E-02 | lm-token-buffer-mib: 1.164335E+02 | global_load_balancing_loss: 1.002033E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11232.557 | params rms: 0.060821 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:22:41.913913 | finish at 2026-10-15 00:49:58 + [2026-10-11 05:29:44] iteration 59/ 2384 | consumed samples: 1888 | elapsed time per iteration (ms): 148187.3 | throughput per GPU (TFLOP/s/GPU): 32.4 | MFU 10.38% | FLOPs estimator: workload-aware | logical tokens/s: 113216.1 | valid tokens/s: 113216.1 | physical tokens/s: 113760.9 | loss tokens/s: 87868.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.485888E-01 | lm-global-supervised-tokens: 1.302101E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.293329E-02 | lm-token-buffer-mib: 1.163440E+02 | global_load_balancing_loss: 1.002617E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11232.507 | params rms: 0.060820 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:42:15.497367 | finish at 2026-10-15 05:12:00 + [2026-10-11 05:32:35] iteration 60/ 2384 | consumed samples: 1920 | elapsed time per iteration (ms): 170820.0 | throughput per GPU (TFLOP/s/GPU): 28.8 | MFU 9.23% | FLOPs estimator: workload-aware | logical tokens/s: 98215.7 | valid tokens/s: 98215.7 | physical tokens/s: 98620.0 | loss tokens/s: 72417.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.134244E-01 | lm-global-supervised-tokens: 1.237036E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.745151E-02 | lm-token-buffer-mib: 1.168145E+02 | global_load_balancing_loss: 1.000587E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11232.455 | params rms: 0.060820 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:16:25.652695 | finish at 2026-10-15 19:49:01 +[rank58]:[E1011 07:33:52.596038254 ProcessGroupNCCL.cpp:685] [Rank 58] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200003 milliseconds before timing out. +[rank59]:[E1011 07:33:52.596036726 ProcessGroupNCCL.cpp:685] [Rank 59] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200004 milliseconds before timing out. +[rank62]:[E1011 07:33:52.598795775 ProcessGroupNCCL.cpp:685] [Rank 62] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200011 milliseconds before timing out. +[rank60]:[E1011 07:33:52.598796484 ProcessGroupNCCL.cpp:685] [Rank 60] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200011 milliseconds before timing out. +[rank60]:[E1011 07:33:52.601794119 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 60] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank58]:[E1011 07:33:52.601791914 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 58] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank59]:[E1011 07:33:52.601796650 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 59] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank62]:[E1011 07:33:52.601797816 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 62] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank60]:[E1011 07:33:52.602219680 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank58]:[E1011 07:33:52.602224268 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank59]:[E1011 07:33:52.602227501 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank62]:[E1011 07:33:52.602229542 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank58]:[E1011 07:33:52.603802974 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 58] First PG on this rank to signal dumping. +[rank60]:[E1011 07:33:52.603807412 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 60] First PG on this rank to signal dumping. +[rank59]:[E1011 07:33:52.603809221 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 59] First PG on this rank to signal dumping. +[rank62]:[E1011 07:33:52.603811849 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 62] First PG on this rank to signal dumping. +[rank57]:[E1011 07:33:52.613883732 ProcessGroupNCCL.cpp:685] [Rank 57] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200026 milliseconds before timing out. +[rank57]:[E1011 07:33:52.613976137 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 57] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank57]:[E1011 07:33:52.613989944 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank57]:[E1011 07:33:52.614058917 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 57] First PG on this rank to signal dumping. +[rank56]:[E1011 07:33:52.614768223 ProcessGroupNCCL.cpp:685] [Rank 56] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200027 milliseconds before timing out. +[rank56]:[E1011 07:33:52.614875477 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 56] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank56]:[E1011 07:33:52.614886787 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank56]:[E1011 07:33:52.614964362 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 56] First PG on this rank to signal dumping. +[rank61]:[E1011 07:33:52.636747827 ProcessGroupNCCL.cpp:685] [Rank 61] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200049 milliseconds before timing out. +[rank61]:[E1011 07:33:52.636841355 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 61] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank61]:[E1011 07:33:52.636856363 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank61]:[E1011 07:33:52.636927812 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 61] First PG on this rank to signal dumping. +[rank63]:[E1011 07:33:52.683553313 ProcessGroupNCCL.cpp:685] [Rank 63] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200096 milliseconds before timing out. +[rank63]:[E1011 07:33:52.683642704 ProcessGroupNCCL.cpp:2252] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 63] failure detected by watchdog at work sequence id: 545 PG status: last enqueued work: 545, last completed work: 544 +[rank63]:[E1011 07:33:52.683656975 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank63]:[E1011 07:33:52.683726697 ProcessGroupNCCL.cpp:2584] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 63] First PG on this rank to signal dumping. +[rank58]:[E1011 07:33:52.944777448 ProcessGroupNCCL.cpp:685] [Rank 58] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200017 milliseconds before timing out. +[rank58]:[E1011 07:33:52.944814239 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 459(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank58]:[E1011 07:33:52.944862229 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank57]:[E1011 07:33:52.968743494 ProcessGroupNCCL.cpp:685] [Rank 57] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200040 milliseconds before timing out. +[rank57]:[E1011 07:33:52.968780797 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 459(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank57]:[E1011 07:33:52.968790690 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank57]:[W1011 07:33:52.968867620 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=100, addr=[cs-3554a-53569-server]:34978, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ff6bb8d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7ff69f9bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7ff69f9bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7ff69f9bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7ff69f9b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7ff65ee95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7ff65ee9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7ff65eea2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7ff65eea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7ff642801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7ff6bcafdaa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7ff6bcb8ac3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank58]:[W1011 07:33:52.947638455 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=100, addr=[cs-3554a-53569-server]:55392, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fe0024d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fdfe65bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fdfe65bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fdfe65bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fdfe65b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fdfa5a95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fdfa5a9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fdfa5aa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fdfa5aa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fdf89401db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fe003710aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fe00379dc3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank57]:[E1011 07:33:52.987753880 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 459(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 1] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ff6bb8d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7ff69f9bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7ff69f9bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7ff69f9bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7ff69f9b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7ff65ee95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7ff65ee9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7ff65eea2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7ff65eea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7ff642801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7ff6bcafdaa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7ff6bcb8ac3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank58]:[E1011 07:33:52.987759518 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 459(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 2] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fe0024d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fdfe65bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fdfe65bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fdfe65bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fdfe65b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fdfa5a95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fdfa5a9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fdfa5aa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fdfa5aa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fdf89401db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fe003710aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fe00379dc3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank59]:[E1011 07:33:52.011611918 ProcessGroupNCCL.cpp:685] [Rank 59] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200085 milliseconds before timing out. +[rank59]:[E1011 07:33:52.011651585 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 459(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank59]:[E1011 07:33:52.011659572 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank60]:[E1011 07:33:52.015161842 ProcessGroupNCCL.cpp:685] [Rank 60] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200086 milliseconds before timing out. +[rank60]:[E1011 07:33:52.015201780 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 459(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank60]:[E1011 07:33:52.015209249 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank56]:[E1011 07:33:52.015482235 ProcessGroupNCCL.cpp:685] [Rank 56] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200087 milliseconds before timing out. +[rank56]:[E1011 07:33:52.015533455 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 459(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank56]:[E1011 07:33:52.015542184 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank62]:[E1011 07:33:52.015890366 ProcessGroupNCCL.cpp:685] [Rank 62] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200089 milliseconds before timing out. +[rank62]:[E1011 07:33:52.015928868 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 459(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank62]:[E1011 07:33:52.015936845 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank59]:[W1011 07:33:52.011737276 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=100, addr=[cs-3554a-53569-server]:55390, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f8fcdbdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f900f5bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f900f5bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f900f5bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f900f5b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f8fcea95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f8fcea9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f8fceaa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f8fceaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f8fb2201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f902c678aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f902c705c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank59]:[E1011 07:33:52.016516535 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 459(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 3] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f8fcdbdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f900f5bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f900f5bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f900f5bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f900f5b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f8fcea95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f8fcea9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f8fceaa2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f8fceaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f8fb2201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f902c678aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f902c705c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank61]:[E1011 07:33:52.019566125 ProcessGroupNCCL.cpp:685] [Rank 61] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200088 milliseconds before timing out. +[rank61]:[E1011 07:33:52.019612394 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 459(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank61]:[E1011 07:33:52.019619864 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank60]:[W1011 07:33:52.015278661 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=100, addr=[cs-3554a-53569-server]:55376, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f97c317eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f97a71bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f97a71bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f97a71bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f97a71b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f9766695880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f976669599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f97666a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f97666a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f974a001db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f97c437faa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f97c440cc3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank60]:[E1011 07:33:52.020113139 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 459(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 4] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f97c317eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f97a71bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f97a71bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f97a71bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f97a71b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f9766695880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f976669599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f97666a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f97666a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f974a001db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f97c437faa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f97c440cc3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank56]:[W1011 07:33:52.015640522 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=100, addr=[cs-3554a-53569-server]:55406, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fe6172d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fe5fb3bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fe5fb3bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fe5fb3bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fe5fb3b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fe5ba895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fe5ba89599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fe5ba8a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fe5ba8a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fe59e201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fe618532aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fe6185bfc3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank56]:[E1011 07:33:52.020673303 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 459(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 0] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7fe6172d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7fe5fb3bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7fe5fb3bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7fe5fb3bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7fe5fb3b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7fe5ba895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7fe5ba89599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7fe5ba8a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7fe5ba8a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7fe59e201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7fe618532aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7fe6185bfc3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank62]:[W1011 07:33:52.016026613 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=100, addr=[cs-3554a-53569-server]:55366, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f24569deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f24983bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f24983bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f24983bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f24983b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f2457895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f245789599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f24578a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f24578a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f243b001db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f24b5467aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f24b54f4c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank62]:[E1011 07:33:52.020934242 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 459(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 6] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f24569deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f24983bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f24983bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f24983bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f24983b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f2457895880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f245789599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f24578a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f24578a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f243b001db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f24b5467aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f24b54f4c3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank61]:[W1011 07:33:52.019696154 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=100, addr=[cs-3554a-53569-server]:55368, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f679377eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f67777bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f67777bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f67777bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f67777b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f6736c95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f6736c9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f6736ca2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f6736ca4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f671a601db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f6794a00aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f6794a8dc3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank61]:[E1011 07:33:52.024482872 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 459(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 5] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f679377eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f67777bb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f67777bbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f67777bd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f67777b71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f6736c95880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f6736c9599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f6736ca2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f6736ca4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f671a601db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f6794a00aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f6794a8dc3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank63]:[E1011 07:33:52.025678288 ProcessGroupNCCL.cpp:685] [Rank 63] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=52327, OpType=_ALLGATHER_BASE, NumelIn=192, NumelOut=1536, Timeout(ms)=7200000) ran for 7200093 milliseconds before timing out. +[rank63]:[E1011 07:33:52.025723870 ProcessGroupNCCL.cpp:2252] [PG ID 14 PG GUID 459(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] failure detected by watchdog at work sequence id: 52327 PG status: last enqueued work: 52327, last completed work: 52326 +[rank63]:[E1011 07:33:52.025732087 ProcessGroupNCCL.cpp:732] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank63]:[W1011 07:33:52.025809985 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=106, addr=[cs-3554a-53569-server]:55416, remote=[10-239-252-55.cs-517b1-57d54-jupyter-svc.default.svc.cluster.local]:35873): Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f0e881deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f0ec9bbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f0ec9bbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f0ec9bbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f0ec9bb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f0e89095880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f0e8909599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f0e890a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f0e890a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f0e6c801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f0ee6c92aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f0ee6d1fc3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank63]:[E1011 07:33:52.030521311 ProcessGroupNCCL.cpp:2574] [PG ID 14 PG GUID 459(EXPERT_TENSOR_AND_MODEL_PARALLEL_GROUP) Rank 7] Failed to broadcast signal exception_dump through TCPStore. Error: Broken pipe +Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f0e881deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0x5d694d1 (0x7f0ec9bbb4d1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #2: + 0x5d69d62 (0x7f0ec9bbbd62 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #3: + 0x5d6b86e (0x7f0ec9bbd86e in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #4: c10d::TCPStore::set(std::__cxx11::basic_string, std::allocator > const&, std::vector > const&) + 0x1be (0x7f0ec9bb71ce in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so) +frame #5: c10d::ProcessGroupNCCL::broadcastSignal(c10::intrusive_ptr >&, std::__cxx11::basic_string, std::allocator > const&, int) + 0x60 (0x7f0e89095880 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #6: c10d::ProcessGroupNCCL::broadcastDumpSignal() + 0x5f (0x7f0e8909599f in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #7: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x7e5 (0x7f0e890a2db5 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #8: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f0e890a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #9: + 0xecdb4 (0x7f0e6c801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #10: + 0x9caa4 (0x7f0ee6c92aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #11: + 0x129c3c (0x7f0ee6d1fc3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank59]:[E1011 07:34:52.805165411 ProcessGroupNCCL.cpp:746] [Rank 59] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank59]:[E1011 07:34:52.805242352 ProcessGroupNCCL.cpp:760] [Rank 59] To avoid data inconsistency, we are taking the entire process down. +[rank57]:[E1011 07:34:52.807080804 ProcessGroupNCCL.cpp:746] [Rank 57] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank57]:[E1011 07:34:52.807099115 ProcessGroupNCCL.cpp:760] [Rank 57] To avoid data inconsistency, we are taking the entire process down. +[rank59]:[E1011 07:34:52.807224150 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 59] Process group watchdog thread terminated with exception: [Rank 59] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200004 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f8fcdbdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f8fceaa0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f8fceaa3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f8fceaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f8fb2201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f902c678aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f902c705c3c in /lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' +[rank57]:[E1011 07:34:52.808026106 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 57] Process group watchdog thread terminated with exception: [Rank 57] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200026 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ff6bb8d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7ff65eea0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7ff65eea3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7ff65eea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7ff642801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7ff6bcafdaa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7ff6bcb8ac3c in /lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 59] Process group watchdog thread terminated with exception: [Rank 59] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200004 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f8fcdbdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f8fceaa0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f8fceaa3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f8fceaa4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f8fb2201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f902c678aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f902c705c3c in /lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f8fcdbdeeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7f8fcea7c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7f8fce5a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7f8fb2201db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7f902c678aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7f902c705c3c in /lib/x86_64-linux-gnu/libc.so.6) + + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 57] Process group watchdog thread terminated with exception: [Rank 57] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200026 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ff6bb8d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7ff65eea0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7ff65eea3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7ff65eea4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7ff642801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7ff6bcafdaa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7ff6bcb8ac3c in /lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ff6bb8d9eb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7ff65ee7c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7ff65e9a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7ff642801db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7ff6bcafdaa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7ff6bcb8ac3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank60]:[E1011 07:34:52.811764640 ProcessGroupNCCL.cpp:746] [Rank 60] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank60]:[E1011 07:34:52.811786963 ProcessGroupNCCL.cpp:760] [Rank 60] To avoid data inconsistency, we are taking the entire process down. +[rank60]:[E1011 07:34:52.812942593 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 60] Process group watchdog thread terminated with exception: [Rank 60] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200011 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f97c317eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f97666a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f97666a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f97666a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f974a001db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f97c437faa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f97c440cc3c in /lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 60] Process group watchdog thread terminated with exception: [Rank 60] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200011 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f97c317eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f97666a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f97666a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f97666a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f974a001db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f97c437faa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f97c440cc3c in /lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f97c317eeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7f976667c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7f97661a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7f974a001db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7f97c437faa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7f97c440cc3c in /lib/x86_64-linux-gnu/libc.so.6) + +[rank62]:[E1011 07:34:52.815240896 ProcessGroupNCCL.cpp:746] [Rank 62] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank62]:[E1011 07:34:52.815262380 ProcessGroupNCCL.cpp:760] [Rank 62] To avoid data inconsistency, we are taking the entire process down. +[rank62]:[E1011 07:34:52.816374551 ProcessGroupNCCL.cpp:2068] [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 62] Process group watchdog thread terminated with exception: [Rank 62] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200011 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f24569deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f24578a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f24578a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f24578a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f243b001db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f24b5467aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f24b54f4c3c in /lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 9 PG GUID 359(TENSOR_AND_DATA_PARALLEL_GROUP_WITH_CP) Rank 62] Process group watchdog thread terminated with exception: [Rank 62] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=545, OpType=ALLREDUCE, NumelIn=10368, NumelOut=10368, Timeout(ms)=7200000) ran for 7200011 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:688 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f24569deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7f24578a0147 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1591 (0x7f24578a3b61 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xd2 (0x7f24578a4ec2 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xecdb4 (0x7f243b001db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x9caa4 (0x7f24b5467aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #6: + 0x129c3c (0x7f24b54f4c3c in /lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2074 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7f24569deeb0 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libc10.so) +frame #1: + 0xe1c1a1 (0x7f245787c1a1 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x9468e6 (0x7f24573a68e6 in /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xecdb4 (0x7f243b001db4 in /lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x9caa4 (0x7f24b5467aa4 in /lib/x86_64-linux-gnu/libc.so.6) +frame #5: + 0x129c3c (0x7f24b54f4c3c in /lib/x86_64-linux-gnu/libc.so.6) + +W1011 07:34:52.762000 470369 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 470429 closing signal SIGTERM +W1011 07:34:52.775000 470369 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 470431 closing signal SIGTERM +W1011 07:34:52.779000 470369 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 470434 closing signal SIGTERM +W1011 07:34:52.783000 470369 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 470436 closing signal SIGTERM +W1011 07:35:22.819000 470369 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 470429 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:25.590000 470369 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 470431 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:28.499000 470369 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 470434 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +W1011 07:35:31.444000 470369 torch/distributed/elastic/multiprocessing/api.py:919] Unable to shutdown process 470436 via Signals.SIGTERM, forcefully exiting via Signals.SIGKILL +E1011 07:35:34.592000 470369 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -6) local_rank: 1 (pid: 470430) of binary: /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +Traceback (most recent call last): + File "/mnt/ssd/yulan/cache/installs/cpython-3.10.19-linux-x86_64-gnu/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/mnt/ssd/yulan/cache/installs/cpython-3.10.19-linux-x86_64-gnu/lib/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 905, in + main() + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 143, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 277, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-10-11_07:34:52 + host : cs-3554a-53569-server + rank : 59 (local_rank: 3) + exitcode : -6 (pid: 470432) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 470432 +[2]: + time : 2026-10-11_07:34:52 + host : cs-3554a-53569-server + rank : 60 (local_rank: 4) + exitcode : -6 (pid: 470433) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 470433 +[3]: + time : 2026-10-11_07:34:52 + host : cs-3554a-53569-server + rank : 62 (local_rank: 6) + exitcode : -6 (pid: 470435) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 470435 +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-10-11_07:34:52 + host : cs-3554a-53569-server + rank : 57 (local_rank: 1) + exitcode : -6 (pid: 470430) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 470430 +============================================================ diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node0.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node0.sh new file mode 100644 index 0000000000000000000000000000000000000000..bfcac16bada2f08495fbeabb06625fb870642418 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node0.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Stage16a (two-phase plan, run A): 40B tokens normal attention on part1_40B split. +# 512K seq length, loads stage15 final weights (resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 32 docs x 524288 = 16,777,216 tokens/step; 76288 samples = 2384 iters. +# part1 = 39.9998B tokens (every source cut at the same doc ratio, see split_20261010/split_report.json). +# BACKUP PLAN resume: CP16+EP16, loads the fully-reshardable ckpt produced by stage16a_reshard_fr_20261011a (runbook in header of reshard_node0.sh). +# After completion, stage16b resumes from this run's final ckpt with sparse attention. +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 0 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 16 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_reshard_fr_20261011a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node1.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node1.sh new file mode 100644 index 0000000000000000000000000000000000000000..71634bd111dcae85cdc354171b2692972a7861c1 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node1.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 1 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 16 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_reshard_fr_20261011a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node2.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node2.sh new file mode 100644 index 0000000000000000000000000000000000000000..c900ab95749ada72908ff67c6b52f806bcb275da --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node2.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 2 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 16 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_reshard_fr_20261011a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node3.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node3.sh new file mode 100644 index 0000000000000000000000000000000000000000..160185dbaf0ecca2a611cdedee68f8ed092033a7 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node3.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 3 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 16 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_reshard_fr_20261011a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node4.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node4.sh new file mode 100644 index 0000000000000000000000000000000000000000..64d9bad0e3407a13d84c3a7c6c34feab474bf522 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node4.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 4 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 16 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_reshard_fr_20261011a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node5.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node5.sh new file mode 100644 index 0000000000000000000000000000000000000000..0ebdf9c104c1c3c8333d51666e29ad031847ef76 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node5.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 5 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 16 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_reshard_fr_20261011a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node6.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node6.sh new file mode 100644 index 0000000000000000000000000000000000000000..557e1227e0537f939e2146f1db84b5b6656c2668 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node6.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 6 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 16 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_reshard_fr_20261011a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node7.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node7.sh new file mode 100644 index 0000000000000000000000000000000000000000..02c5b83938201334d3a10a92b44f0f1fecac7561 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_cp16ep16_node7.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 7 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 16 --expert-model-parallel-size 16 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_reshard_fr_20261011a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node0.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node0.log new file mode 100644 index 0000000000000000000000000000000000000000..4154928f9914126746b73ddd69d1114c390e7974 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node0.log @@ -0,0 +1,4863 @@ +W1011 16:05:40.453000 738516 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 16:05:40.453000 738516 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +using world size: 64, data-parallel size: 2, context-parallel size: 32, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:HuggingFaceTokenizer +Number of virtual stages per pipeline stage: None +[gdn-fused-split-forward] optimizer overlap enabled: param-gather=ddp-extra-param-bridge grad-reduce=standard-autograd-hooks +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/training/utils.py:548: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn(message) +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... soft + activation_func_clamp_value ..................... 7.0 + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ False + adam_eps ........................................ 1e-15 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... True + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... True + auto_generate_cu_seqlens ........................ True + barrier_with_L1_time ............................ False + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ True + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 32 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... True + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. False + cross_entropy_fusion_impl ....................... linear_triton + cross_entropy_loss_fusion ....................... True + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 2 + data_path ....................................... ['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. cyclic + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. 40000000 + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... False + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 120 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... obfd + document_packing_dp_size ........................ None + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 524288 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 99999999 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... True + exit_signal_handler ............................. True + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... True + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ 30.0 + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... True + gdn_gated_norm_backend .......................... linghe + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 32 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... True + hybrid_attention_ratio .......................... 0.0625 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.5 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 64 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. True + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 32 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... global-token + load ............................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + load_complemental_dataset ....................... ['loss_mask'] + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... ['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] + log_interval .................................... 1 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ 10 + log_moe_activations ............................. True + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... ['attention::linear_qkv', 'attention::o_proj', 'mlp::linear_fc1', 'mlp::linear_fc2', 'input_layernorm', 'pre_mlp_layernorm', 'embedding', 'lm_head', 'gated_delta::in_proj', 'gated_delta::in_proj_bias', 'gated_delta::conv1d', 'gated_delta::conv1d_bias', 'gated_delta::dt_bias', 'gated_delta::A_log', 'gated_delta::out_norm', 'gated_delta::out_norm_bias', 'gated_delta::out_proj', 'gated_delta::out_proj_bias'] + log_params_norm ................................. True + log_per_layer_grad_rms .......................... True + log_per_module_grad_rms ......................... True + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_update_rms .................................. True + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0005 + lr_decay_iters .................................. None + lr_decay_samples ................................ 76288 + lr_decay_style .................................. constant + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 524288 + max_seqlen_per_cp_rank .......................... None + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 0.0005 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.05 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. True + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... True + moe_router_gemm_backend ......................... te + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. global_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... True + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... gated_delta_net + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. 0 + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ True + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 4 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... False + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. 8 + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... full + recompute_method ................................ block + recompute_modules ............................... None + recompute_num_layers ............................ 56 + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... True + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit + sample_rate ..................................... 1.0 + save ............................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + save_after_load ................................. False + save_interval ................................... 40 + save_retain_interval ............................ 1000000000 + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 524288 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... 999,1,0 + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... 0 + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... all + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... None + train_samples ................................... 76288 + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. True + use_fused_weighted_squared_relu ................. False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. stage15_115B_8node_gdnlralpha_off_20261003a + wandb_project ................................... qwen2.5_3b_mom_cpt + wandb_save_dir .................................. /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 64 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 16 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 1 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 6 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 5 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 0 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 3 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank1]:[W1011 16:06:36.940292487 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank5]:[W1011 16:06:36.940296597 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank6]:[W1011 16:06:36.940295406 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank0]:[W1011 16:06:36.940299516 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank1]:[W1011 16:06:36.940306234 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank5]:[W1011 16:06:36.940308860 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank6]:[W1011 16:06:36.940311522 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank0]:[W1011 16:06:36.940313083 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank3]:[W1011 16:06:36.940968301 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank3]:[W1011 16:06:36.940985217 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 7 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 2 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank7]:[W1011 16:06:36.941506592 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank7]:[W1011 16:06:36.941518249 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank2]:[W1011 16:06:36.941534321 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank2]:[W1011 16:06:36.941546624 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 4 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank4]:[W1011 16:06:36.944468125 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank4]:[W1011 16:06:36.944478323 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank [Gloo] Rank 0 is connected to 01 is connected to peer ranks. [Gloo] Rank 1[Gloo] Rank Expected number of connected peer ranks is : peer ranks. 0[Gloo] Rank 10Expected number of connected peer ranks is : is connected to 0 + is connected to 11 + is connected to 1 peer ranks. 1 peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 1Expected number of connected peer ranks is : 1 +1 + +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to [Gloo] Rank 70 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +> compiling dataset index builder ... +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank4]:[W1011 16:06:37.239404977 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W1011 16:06:37.239410335 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W1011 16:06:37.239411122 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank6]:[W1011 16:06:37.239808130 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank1]:[W1011 16:06:37.240785139 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank2]:[W1011 16:06:37.241004837 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank7]:[W1011 16:06:37.241585120 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.138 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank0]:[W1011 16:06:37.380845575 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +NCCL version 2.27.3+cuda12.9 +[rank2]:[W1011 16:06:38.514582086 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank7]:[W1011 16:06:38.514598180 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank2]:[W1011 16:06:38.514647894 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank7]:[W1011 16:06:38.514679625 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank6]:[W1011 16:06:38.514917659 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank0]:[W1011 16:06:38.514918674 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank5]:[W1011 16:06:38.514927445 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank4]:[W1011 16:06:38.514949370 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank6]:[W1011 16:06:38.514969830 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank3]:[W1011 16:06:38.514971744 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank0]:[W1011 16:06:38.514999244 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank1]:[W1011 16:06:38.515009934 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank5]:[W1011 16:06:38.515024967 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank4]:[W1011 16:06:38.515051713 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank3]:[W1011 16:06:38.515085080 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank1]:[W1011 16:06:38.515122960 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +>>> done with compiling and loading fused kernels. Compilation time: 2.296 seconds +time to initialize megatron (seconds): 366.348 +[after megatron is initialized] datetime: 2026-10-11 16:11:56 +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5,569,877,120 + > number of trainable parameters on (tensor, pipeline) model parallel rank (0, 0): 5,569,877,120 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, defer_grad_checks=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_megatron_fsdp=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache=False, nccl_ub=False, fsdp_double_buffer=False, outer_dp_sharding_strategy='no_shard', disable_symmetric_registration=False, delay_wgrad_compute=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 28 +Params for bucket 1 (202899456 elements, 202899456 padded size): + module.output_layer.weight torch.Size([99072, 2048]) +Params for bucket 2 (40222848 elements, 40222848 padded size): + module.decoder.final_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.final_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.55.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.54.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.54.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.54.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.54.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.55.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.55.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.54.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.final_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.55.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.54.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.54.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.55.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.55.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.55.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.54.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.54.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.55.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.55.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.54.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.final_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.55.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.54.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.54.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.54.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.55.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.55.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.55.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.55.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.54.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.54.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.55.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.55.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.54.self_attention.k_proj.weight torch.Size([512, 2048]) +Params for bucket 3 (41666688 elements, 41666688 padded size): + module.decoder.layers.53.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.52.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.53.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.53.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.52.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.52.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.52.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.53.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.52.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.53.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.53.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.53.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.52.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.51.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.51.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.53.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.53.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.52.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.52.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.52.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.51.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.51.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.53.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.52.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.53.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.53.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.53.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.53.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.52.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.52.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.52.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.52.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.53.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.53.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.52.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.52.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) +Params for bucket 4 (43304064 elements, 43304064 padded size): + module.decoder.layers.51.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.49.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.51.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.51.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.50.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.50.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.50.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.49.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.51.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.50.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.50.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.50.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.51.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.50.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.50.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.49.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.51.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.50.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.50.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.50.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.51.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.50.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.49.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.51.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.51.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.51.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.51.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.50.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.50.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.50.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.51.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.50.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.49.mlp.shared_experts.gate_weight torch.Size([1, 2048]) +Params for bucket 5 (40014912 elements, 40014976 padded size): + module.decoder.layers.48.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.48.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.47.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.49.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.48.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.48.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.47.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.47.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.47.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.49.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.48.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.47.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.47.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.47.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.49.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.48.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.48.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.47.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.49.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.47.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.48.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.49.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.48.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.47.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.48.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.47.self_attention.a_proj.weight torch.Size([32, 2048]) +Params for bucket 6 (41133120 elements, 41133184 padded size): + module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.46.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.45.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.45.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.45.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.45.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.46.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.45.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.45.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.46.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.46.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.45.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.45.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.47.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.47.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.47.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.46.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.45.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.45.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.45.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.45.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.47.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.46.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.46.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.45.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.46.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.45.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.47.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.46.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.46.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.45.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.45.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) +Params for bucket 7 (41666688 elements, 41666688 padded size): + module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.44.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.43.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.43.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.44.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.43.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.43.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.44.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.44.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.43.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.43.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.44.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.44.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.43.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.44.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.44.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.44.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.43.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.43.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.42.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.42.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.44.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.44.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.43.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.43.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.42.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.42.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.44.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.43.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.43.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.44.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.44.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.44.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.44.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.43.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.43.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.43.self_attention.z_proj.weight torch.Size([2048, 2048]) +Params for bucket 8 (41345216 elements, 41345280 padded size): + module.decoder.layers.42.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.41.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.41.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.41.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.40.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.40.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.42.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.41.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.40.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.42.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.42.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.41.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.41.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.42.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.41.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.41.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.40.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.40.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.42.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.41.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.40.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.40.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.42.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.41.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.41.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.41.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.40.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.42.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.41.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.41.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.41.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.40.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.42.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.42.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.42.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.42.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.41.self_attention.k_proj.weight torch.Size([512, 2048]) +Params for bucket 9 (41201792 elements, 41201792 padded size): + module.decoder.layers.39.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.39.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.39.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.39.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.38.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.39.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.38.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.39.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.38.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.40.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.39.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.39.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.38.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.38.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.38.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.40.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.39.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.38.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.38.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.40.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.39.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.39.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.39.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.40.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.40.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.39.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.39.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.39.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.38.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.40.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.40.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.39.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.38.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) +Params for bucket 10 (40222848 elements, 40222848 padded size): + module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.37.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.37.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.37.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.36.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.38.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.37.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.37.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.36.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.36.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.37.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.36.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.38.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.37.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.37.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.36.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.36.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.36.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.37.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.37.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.36.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.36.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.38.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.37.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.37.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.36.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.38.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.38.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.37.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.37.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.36.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.36.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.36.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.36.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.38.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.37.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.37.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.36.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.36.input_layernorm.gate_down.weight torch.Size([16, 2048]) +Params for bucket 11 (43763840 elements, 43763840 padded size): + module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.35.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.34.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.33.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.35.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.34.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.34.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.34.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.34.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.35.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.35.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.34.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.35.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.34.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.34.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.35.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.35.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.35.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.34.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.34.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.33.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.33.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.35.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.35.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.35.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.34.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.33.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.35.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.34.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.34.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.34.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.35.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.35.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.35.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.35.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.34.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.34.self_attention.v_proj.weight torch.Size([2048, 2048]) +Params for bucket 12 (41345216 elements, 41345280 padded size): + module.decoder.layers.33.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.32.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.32.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.31.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.31.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.33.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.32.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.32.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.32.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.31.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.33.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.33.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.32.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.32.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.31.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.31.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.33.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.32.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.32.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.32.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.31.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.33.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.32.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.33.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.32.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.32.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.31.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.33.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.32.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.33.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.33.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.33.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.33.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.32.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.32.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.31.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.31.self_attention.out_norm.weight torch.Size([64]) +Params for bucket 13 (41201792 elements, 41201792 padded size): + module.decoder.layers.31.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.30.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.29.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.30.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.30.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.29.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.29.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.29.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.29.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.30.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.31.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.30.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.30.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.30.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.29.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.31.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.30.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.30.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.30.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.29.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.31.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.31.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.31.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.30.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.29.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.31.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.30.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.30.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.30.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.30.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.29.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.30.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.29.self_attention.b_proj.weight torch.Size([32, 2048]) +Params for bucket 14 (40222848 elements, 40222848 padded size): + module.decoder.layers.29.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.28.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.27.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.28.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.28.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.28.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.28.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.27.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.29.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.29.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.28.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.28.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.27.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.27.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.29.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.29.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.28.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.27.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.27.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.28.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.28.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.27.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.27.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.28.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.27.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.27.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.27.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.28.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.28.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.28.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.27.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.29.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.28.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.28.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.27.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.27.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.27.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.27.input_layernorm.norm.weight torch.Size([2048]) +Params for bucket 15 (43763840 elements, 43763840 padded size): + module.decoder.layers.26.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.25.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.25.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.25.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.24.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.24.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.26.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.26.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.26.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.26.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.25.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.24.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.24.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.26.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.26.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.25.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.25.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.25.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.25.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.26.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.26.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.25.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.26.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.26.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.25.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.26.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.25.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.25.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.26.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.26.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.25.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.26.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.26.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.25.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.25.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.25.self_attention.z_proj.weight torch.Size([2048, 2048]) +Params for bucket 16 (41206912 elements, 41206912 padded size): + module.decoder.layers.24.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.23.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.23.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.23.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.22.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.24.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.23.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.22.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.24.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.23.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.22.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.24.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.24.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.24.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.24.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.23.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.22.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.22.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.24.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.22.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.24.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.23.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.24.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.24.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.23.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.22.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.22.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.24.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.23.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.23.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.22.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) +Params for bucket 17 (43022336 elements, 43022336 padded size): + module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.21.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.20.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.22.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.20.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.20.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.21.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.21.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.21.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.20.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.22.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.21.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.21.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.22.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.21.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.22.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.22.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.22.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.21.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.20.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.22.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.21.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.21.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.20.self_attention.linear_qkv.bias torch.Size([5120]) +Params for bucket 18 (40222848 elements, 40222848 padded size): + module.decoder.layers.19.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.18.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.18.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.20.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.19.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.19.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.18.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.20.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.19.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.19.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.19.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.18.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.18.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.18.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.20.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.19.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.18.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.18.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.18.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.20.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.19.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.19.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.19.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.19.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.18.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.19.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.18.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.18.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.18.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.18.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.19.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.19.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.18.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.19.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.19.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.18.self_attention.out_norm.weight torch.Size([64]) +Params for bucket 19 (41666688 elements, 41666688 padded size): + module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.17.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.17.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.16.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.16.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.16.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.17.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.17.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.16.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.16.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.16.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.15.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.15.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.17.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.17.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.17.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.17.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.16.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.15.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.15.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.17.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.16.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.16.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.16.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.16.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.17.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.16.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.17.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.17.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.16.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.17.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.16.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.16.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.17.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.17.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.17.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.16.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) +Params for bucket 20 (41345216 elements, 41345280 padded size): + module.decoder.layers.15.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.14.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.15.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.14.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.14.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.13.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.13.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.15.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.14.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.13.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.15.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.14.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.14.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.14.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.14.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.13.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.13.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.15.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.15.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.14.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.14.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.13.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.15.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.14.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.13.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.15.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.15.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.14.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.14.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.14.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.13.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.13.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.15.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.15.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.15.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.14.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.14.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) +Params for bucket 21 (41063488 elements, 41063552 padded size): + module.decoder.layers.13.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.13.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.11.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.11.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.11.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.13.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.12.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.11.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.11.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.11.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.12.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.12.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.12.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.12.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.11.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.13.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.12.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.12.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.13.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.12.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.11.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.12.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.11.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.13.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.13.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.12.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.11.self_attention.b_proj.weight torch.Size([32, 2048]) +Params for bucket 22 (40222848 elements, 40222848 padded size): + module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.10.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.9.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.9.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.10.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.10.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.10.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.10.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.10.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.9.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.9.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.9.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.11.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.10.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.10.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.9.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.9.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.11.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.11.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.11.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.10.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.9.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.11.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.10.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.10.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.9.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.9.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.9.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.10.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.9.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.9.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.11.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.10.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.10.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.9.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.9.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.10.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.10.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.9.input_layernorm.gate_up.bias torch.Size([2048]) +Params for bucket 23 (43763840 elements, 43763840 padded size): + module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.8.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.7.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.7.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.7.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.6.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.8.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.7.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.7.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.8.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.8.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.7.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.7.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.7.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.7.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.8.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.8.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.8.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.8.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.7.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.6.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.8.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.8.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.8.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.7.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.7.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.6.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.8.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.7.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.7.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.6.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.8.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.8.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.8.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.8.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.7.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.7.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) +Params for bucket 24 (41345216 elements, 41345280 padded size): + module.decoder.layers.6.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.5.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.5.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.5.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.5.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.4.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.6.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.5.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.4.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.6.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.6.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.5.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.4.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.6.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.5.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.5.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.4.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.4.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.4.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.6.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.5.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.4.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.6.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.5.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.5.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.5.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.6.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.5.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.5.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.5.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.4.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.6.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.6.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.6.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.6.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.5.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.4.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) +Params for bucket 25 (41201792 elements, 41201792 padded size): + module.decoder.layers.3.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.3.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.3.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.3.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.4.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.3.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.2.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.3.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.2.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.3.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.3.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.2.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.2.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.4.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.3.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.2.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.2.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.2.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.4.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.3.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.3.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.3.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.2.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.2.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.4.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.4.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.4.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.3.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.3.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.3.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.4.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.3.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.2.self_attention.conv1d.weight torch.Size([3072, 1, 4]) +Params for bucket 26 (40259648 elements, 40259712 padded size): + module.decoder.layers.2.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.2.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.2.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.2.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.1.mlp.linear_fc1.weight torch.Size([9600, 2048]) + module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.1.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.1.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.1.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.1.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.2.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.2.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.1.mlp.linear_fc2.weight torch.Size([2048, 4800]) + module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.1.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.1.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.1.self_attention.z_proj.weight torch.Size([2048, 2048]) +Params for bucket 27 (40116224 elements, 40116224 padded size): + module.decoder.layers.1.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.1.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.1.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.0.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.1.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.1.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.1.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.1.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.0.mlp.linear_fc2.weight torch.Size([2048, 4800]) + module.decoder.layers.0.mlp.linear_fc1.weight torch.Size([9600, 2048]) + module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.0.self_attention.out_proj.weight torch.Size([2048, 2048]) +Params for bucket 28 (213598272 elements, 213598336 padded size): + module.decoder.layers.0.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.0.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.0.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.0.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.0.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.0.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.0.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.0.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.0.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.0.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.0.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.0.input_layernorm.norm.weight torch.Size([2048]) + module.embedding.word_embeddings.weight torch.Size([99072, 2048]) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (3136 elements, 6272 padded size): + module.decoder.layers.47.self_attention.A_log torch.Size([32]) + module.decoder.layers.37.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.18.self_attention.A_log torch.Size([32]) + module.decoder.layers.51.self_attention.A_log torch.Size([32]) + module.decoder.layers.38.self_attention.A_log torch.Size([32]) + module.decoder.layers.34.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.14.self_attention.A_log torch.Size([32]) + module.decoder.layers.10.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.43.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.35.self_attention.A_log torch.Size([32]) + module.decoder.layers.7.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.24.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.16.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.28.self_attention.A_log torch.Size([32]) + module.decoder.layers.25.self_attention.A_log torch.Size([32]) + module.decoder.layers.17.self_attention.A_log torch.Size([32]) + module.decoder.layers.22.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.55.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.40.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.30.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.45.self_attention.A_log torch.Size([32]) + module.decoder.layers.42.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.27.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.6.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.39.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.32.self_attention.A_log torch.Size([32]) + module.decoder.layers.31.self_attention.A_log torch.Size([32]) + module.decoder.layers.3.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.53.self_attention.A_log torch.Size([32]) + module.decoder.layers.13.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.4.self_attention.A_log torch.Size([32]) + module.decoder.layers.0.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.37.self_attention.A_log torch.Size([32]) + module.decoder.layers.29.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.10.self_attention.A_log torch.Size([32]) + module.decoder.layers.34.self_attention.A_log torch.Size([32]) + module.decoder.layers.19.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.15.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.43.self_attention.A_log torch.Size([32]) + module.decoder.layers.7.self_attention.A_log torch.Size([32]) + module.decoder.layers.16.self_attention.A_log torch.Size([32]) + module.decoder.layers.24.self_attention.A_log torch.Size([32]) + module.decoder.layers.50.self_attention.A_log torch.Size([32]) + module.decoder.layers.36.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.44.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.33.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.8.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.2.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.54.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.41.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.5.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.55.self_attention.A_log torch.Size([32]) + module.decoder.layers.52.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.30.self_attention.A_log torch.Size([32]) + module.decoder.layers.26.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.22.self_attention.A_log torch.Size([32]) + module.decoder.layers.9.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.42.self_attention.A_log torch.Size([32]) + module.decoder.layers.40.self_attention.A_log torch.Size([32]) + module.decoder.layers.27.self_attention.A_log torch.Size([32]) + module.decoder.layers.6.self_attention.A_log torch.Size([32]) + module.decoder.layers.1.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.39.self_attention.A_log torch.Size([32]) + module.decoder.layers.11.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.3.self_attention.A_log torch.Size([32]) + module.decoder.layers.47.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.13.self_attention.A_log torch.Size([32]) + module.decoder.layers.18.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.9.self_attention.A_log torch.Size([32]) + module.decoder.layers.0.self_attention.A_log torch.Size([32]) + module.decoder.layers.14.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.51.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.50.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.38.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.29.self_attention.A_log torch.Size([32]) + module.decoder.layers.35.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.19.self_attention.A_log torch.Size([32]) + module.decoder.layers.15.self_attention.A_log torch.Size([32]) + module.decoder.layers.36.self_attention.A_log torch.Size([32]) + module.decoder.layers.28.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.44.self_attention.A_log torch.Size([32]) + module.decoder.layers.33.self_attention.A_log torch.Size([32]) + module.decoder.layers.25.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.17.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.8.self_attention.A_log torch.Size([32]) + module.decoder.layers.2.self_attention.A_log torch.Size([32]) + module.decoder.layers.54.self_attention.A_log torch.Size([32]) + module.decoder.layers.41.self_attention.A_log torch.Size([32]) + module.decoder.layers.5.self_attention.A_log torch.Size([32]) + module.decoder.layers.52.self_attention.A_log torch.Size([32]) + module.decoder.layers.45.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.26.self_attention.A_log torch.Size([32]) + module.decoder.layers.32.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.31.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.1.self_attention.A_log torch.Size([32]) + module.decoder.layers.11.self_attention.A_log torch.Size([32]) + module.decoder.layers.53.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.4.self_attention.dt_bias torch.Size([32]) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 54 +Params for bucket 1 (75497472 elements, 75497472 padded size): + module.decoder.layers.55.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.55.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 2 (75497472 elements, 75497472 padded size): + module.decoder.layers.54.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.54.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 3 (75497472 elements, 75497472 padded size): + module.decoder.layers.53.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.53.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 4 (75497472 elements, 75497472 padded size): + module.decoder.layers.52.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.52.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 5 (75497472 elements, 75497472 padded size): + module.decoder.layers.51.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.51.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 6 (75497472 elements, 75497472 padded size): + module.decoder.layers.50.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.50.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 7 (75497472 elements, 75497472 padded size): + module.decoder.layers.49.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.49.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 8 (75497472 elements, 75497472 padded size): + module.decoder.layers.48.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.48.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 9 (75497472 elements, 75497472 padded size): + module.decoder.layers.47.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.47.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 10 (75497472 elements, 75497472 padded size): + module.decoder.layers.46.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.46.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 11 (75497472 elements, 75497472 padded size): + module.decoder.layers.45.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.45.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 12 (75497472 elements, 75497472 padded size): + module.decoder.layers.44.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.44.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 13 (75497472 elements, 75497472 padded size): + module.decoder.layers.43.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.43.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 14 (75497472 elements, 75497472 padded size): + module.decoder.layers.42.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.42.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 15 (75497472 elements, 75497472 padded size): + module.decoder.layers.41.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.41.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 16 (75497472 elements, 75497472 padded size): + module.decoder.layers.40.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.40.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 17 (75497472 elements, 75497472 padded size): + module.decoder.layers.39.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.39.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 18 (75497472 elements, 75497472 padded size): + module.decoder.layers.38.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.38.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 19 (75497472 elements, 75497472 padded size): + module.decoder.layers.37.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.37.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 20 (75497472 elements, 75497472 padded size): + module.decoder.layers.36.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.36.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 21 (75497472 elements, 75497472 padded size): + module.decoder.layers.35.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.35.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 22 (75497472 elements, 75497472 padded size): + module.decoder.layers.34.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.34.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 23 (75497472 elements, 75497472 padded size): + module.decoder.layers.33.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.33.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 24 (75497472 elements, 75497472 padded size): + module.decoder.layers.32.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.32.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 25 (75497472 elements, 75497472 padded size): + module.decoder.layers.31.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.31.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 26 (75497472 elements, 75497472 padded size): + module.decoder.layers.30.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.30.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 27 (75497472 elements, 75497472 padded size): + module.decoder.layers.29.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.29.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 28 (75497472 elements, 75497472 padded size): + module.decoder.layers.28.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.28.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 29 (75497472 elements, 75497472 padded size): + module.decoder.layers.27.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.27.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 30 (75497472 elements, 75497472 padded size): + module.decoder.layers.26.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.26.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 31 (75497472 elements, 75497472 padded size): + module.decoder.layers.25.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.25.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 32 (75497472 elements, 75497472 padded size): + module.decoder.layers.24.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.24.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 33 (75497472 elements, 75497472 padded size): + module.decoder.layers.23.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.23.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 34 (75497472 elements, 75497472 padded size): + module.decoder.layers.22.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.22.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 35 (75497472 elements, 75497472 padded size): + module.decoder.layers.21.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.21.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 36 (75497472 elements, 75497472 padded size): + module.decoder.layers.20.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.20.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 37 (75497472 elements, 75497472 padded size): + module.decoder.layers.19.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.19.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 38 (75497472 elements, 75497472 padded size): + module.decoder.layers.18.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.18.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 39 (75497472 elements, 75497472 padded size): + module.decoder.layers.17.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.17.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 40 (75497472 elements, 75497472 padded size): + module.decoder.layers.16.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.16.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 41 (75497472 elements, 75497472 padded size): + module.decoder.layers.15.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.15.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 42 (75497472 elements, 75497472 padded size): + module.decoder.layers.14.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.14.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 43 (75497472 elements, 75497472 padded size): + module.decoder.layers.13.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.13.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 44 (75497472 elements, 75497472 padded size): + module.decoder.layers.12.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.12.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 45 (75497472 elements, 75497472 padded size): + module.decoder.layers.11.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.11.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 46 (75497472 elements, 75497472 padded size): + module.decoder.layers.10.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.10.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 47 (75497472 elements, 75497472 padded size): + module.decoder.layers.9.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.9.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 48 (75497472 elements, 75497472 padded size): + module.decoder.layers.8.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.8.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 49 (75497472 elements, 75497472 padded size): + module.decoder.layers.7.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.7.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 50 (75497472 elements, 75497472 padded size): + module.decoder.layers.6.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.6.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 51 (75497472 elements, 75497472 padded size): + module.decoder.layers.5.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.5.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 52 (75497472 elements, 75497472 padded size): + module.decoder.layers.4.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.4.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 53 (75497472 elements, 75497472 padded size): + module.decoder.layers.3.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.3.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 54 (75497472 elements, 75497472 padded size): + module.decoder.layers.2.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.2.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +[GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.0, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.0, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.0, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.0, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.0, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.0, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): FusedLinearCrossEntropyColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +)] +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0005, min_lr=0.0005, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, corrected_weight_decay=True, fp8_recipe='delayed', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.95, adam_eps=1e-15, adam_bias_correction=False, decoupled_weight_decay=True, adamw_lr_mup_scaler=False, sgd_momentum=0.9, adamh_scope='lm_head', adamh_radius=None, adamh_eps=1e-08, spectra_scope='matrices', spectra_hyperball=False, spectra_clip=1.0, spectra_grad_clip=None, spectra_ns_steps=10, muon_momentum=0.9, muon_split_qkv=True, muon_qkv_split_mode='component', muon_split_fc1=True, muon_use_nesterov=False, muon_scale_mode='spectral_mup', muon_fp32_matmul_prec='medium', muon_num_ns_steps=5, muon_tp_mode='blockwise', muon_extra_scale_factor=1.0, muon_split_moe_experts=True, muon_vectorize=[], muon_vectorize_attn_dim='hidden_size', muon_scale_vectorized_mode='full', spectral_ball_momentum=0.9, spectral_ball_use_nesterov=True, spectral_ball_split_qkv=True, spectral_ball_qkv_split_mode='component', spectral_ball_split_fc1=True, spectral_ball_split_moe_experts=True, spectral_ball_msign_steps=8, spectral_ball_solver='bisection', spectral_ball_solver_tolerance_f=1e-08, spectral_ball_solver_max_iterations=20, spectral_ball_radius_mode='spectral_mup', spectral_ball_power_iteration_steps=20, spectral_ball_scale_mode='spectral_mup', spectral_ball_retract_mode='hard', spectral_ball_retract_alpha=0.05, muon_ball_momentum=0.9, muon_ball_use_nesterov=True, muon_ball_split_qkv=True, muon_ball_qkv_split_mode='component', muon_ball_split_fc1=True, muon_ball_split_moe_experts=True, muon_ball_msign_steps=5, muon_ball_radius_mode='spectral_mup', muon_ball_power_iteration_steps=10, muon_ball_scale_mode='spectral_mup', muon_ball_retract_mode='hard', muon_ball_retract_alpha=0.05, use_distributed_optimizer=True, overlap_param_gather=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, log_per_module_update_rms=False, log_per_module_grad_rms=True, barrier_with_L1_time=False, timers=, config_logger_dir='') +setting training iterations to 2384 +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: constant +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints at iteration 40 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer_param_scheduler: > using checkpoint value 0.0005 for learning rate +INFO:megatron.core.optimizer_param_scheduler: > using checkpoint value 0.0005 for minimum learning rate +INFO:megatron.core.optimizer_param_scheduler: > using checkpoint value 0 for warmup iterations +INFO:megatron.core.optimizer_param_scheduler: > using checkpoint value 76288 for total number of iterations +INFO:megatron.core.optimizer_param_scheduler: > using checkpoint value constant for learning rate decay style +INFO:megatron.core.optimizer_param_scheduler: > using checkpoint value 0.1 for start weight decay +INFO:megatron.core.optimizer_param_scheduler: > using checkpoint value 0.1 for end weight decay +INFO:megatron.core.optimizer_param_scheduler: > using checkpoint value 76288 for total number of weight decay iterations +INFO:megatron.core.optimizer_param_scheduler: > using checkpoint value constant for weight decay incr style +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] at iteration 40 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-10-11 16:14:10 +> building train, validation, and test datasets ... + train: 76288 + validation: 320 + test: 320 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 0.999), (0.999, 1.0), None] +> building train, validation, and test datasets for GPT ... +Initializing GlobIndexedDataset +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(76288, 320, 320) and config=GPTDatasetConfig(random_seed=1234, sequence_length=524288, blend=(['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp'], None), blend_per_split=None, multiple_validation_sets=False, full_validation=False, split='999,1,0', split_matrix=[(0, 0.999), (0.999, 1.0), None], num_dataset_builder_threads=1, path_to_cache='/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B', mmap_bin_files=True, fast_cache_load=False, defer_npy_index_mmap=False, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, allow_ambiguous_pad_tokens=False, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=False, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None, load_complemental_dataset=['loss_mask'], document_packing_algorithm='obfd', packing_tokens_per_shard=100000000000, enable_debug_logging=False, emit_document_packing_metadata=True, start_samples=0, context_parallel_size=32, data_parallel_size=2, packing_dp_size=None, sequence_parallel_size=0, hybrid_context_parallel=False) +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 2031 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 2031 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 15046 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 15046 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 3299 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 3299 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 9228 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 9228 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 166555 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 166555 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 7133 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 7133 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 25714 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 25714 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 86039 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 86039 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 11193 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 11193 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 63250 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 63250 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 36116 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 36116 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 527 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 527 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 59344 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 59344 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 294 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 294 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 484 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 484 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 122883 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 122883 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 163 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 163 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 63156 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 63156 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 778640 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 778640 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 43604 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 43604 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 175670 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 175670 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 226991 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 226991 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 8162 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 8162 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 12434 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 12434 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 28587 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 28587 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 199602 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 199602 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 2784740 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 2784740 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1829337 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1829337 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 178682 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 178682 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 52867 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 52867 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 498 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 498 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 177418 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 177418 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 519 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 519 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 78 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 78 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 84548 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 84548 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 16160 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 16160 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 93319 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 93319 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 12473 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 12473 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 228187 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 228187 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 68576 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 68576 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 189 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 189 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1647592 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1647592 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 89874 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 89874 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 373365 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 373365 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 604 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 604 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 86064 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 86064 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 17995 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 17995 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 151034 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 151034 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1849 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1849 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 148 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 148 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 35370 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 35370 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 274 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 274 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 134522 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 134522 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 79269 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 79269 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 432 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 432 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 145607 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 145607 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 579 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 579 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 31211 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 31211 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 287047 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 287047 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 130201 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 130201 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 70545 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 70545 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 104516 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 104516 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 52569 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 52569 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 413 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 413 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 39658 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 39658 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 19381 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 19381 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 263497 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 263497 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 19431 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 19431 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 69929 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 69929 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 24205 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 24205 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 142217 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 142217 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 70554 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 70554 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 260245 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 260245 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 16637 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 16637 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 10723 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 10723 +>>> start_samples: 0 +INFO:megatron.core.datasets.gpt_dataset: > Found complemental loss_mask dataset. Loading from: /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_loss_mask_tmp +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 2031 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 2031 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 15046 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 15046 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 3299 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 3299 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 9228 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 9228 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 166555 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 166555 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 7133 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 7133 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 25714 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 25714 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 86039 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 86039 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 11193 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 11193 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 63250 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 63250 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 36116 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 36116 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 527 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 527 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 59344 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 59344 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 294 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 294 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 484 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 484 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 122883 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 122883 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 163 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 163 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 63156 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 63156 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 778640 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 778640 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 43604 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 43604 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 175670 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 175670 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 226991 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 226991 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 8162 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 8162 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 12434 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 12434 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 28587 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 28587 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 199602 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 199602 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 2784740 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 2784740 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1829337 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1829337 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 178682 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 178682 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 52867 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 52867 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 498 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 498 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 177418 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 177418 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 519 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 519 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 78 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 78 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 84548 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 84548 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 16160 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 16160 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 93319 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 93319 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 12473 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 12473 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 228187 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 228187 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 68576 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 68576 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 189 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 189 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1647592 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1647592 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 89874 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 89874 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 373365 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 373365 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 604 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 604 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 86064 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 86064 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 17995 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 17995 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 151034 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 151034 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1849 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1849 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 148 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 148 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 35370 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 35370 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 274 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 274 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 134522 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 134522 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 79269 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 79269 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 432 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 432 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 145607 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 145607 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 579 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 579 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 31211 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 31211 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 287047 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 287047 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 130201 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 130201 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 70545 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 70545 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 104516 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 104516 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 52569 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 52569 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 413 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 413 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 39658 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 39658 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 19381 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 19381 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 263497 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 263497 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 19431 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 19431 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 69929 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 69929 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 24205 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 24205 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 142217 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 142217 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 70554 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 70554 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 260245 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 260245 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 16637 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 16637 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 10723 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 10723 +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +INFO:megatron.core.datasets.gpt_dataset:{ + "class": "GPTDataset", + "dataset_path": "/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp", + "num_samples": 76288, + "index_split": "train", + "random_seed": 1234, + "sequence_length": 524288, + "split": "999,1,0", + "split_matrix": [ + [ + 0, + 0.999 + ], + [ + 0.999, + 1.0 + ], + null + ], + "tokenizer": { + "class": "megatron.core.tokenizers.text.models.default_tokenizer.DefaultTokenizerText", + "tokenizer_path": "/mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp", + "vocab_file": "None", + "merges_file": "None" + } +} +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 145599 +>>> start_samples: 0 +>>> start_samples: 0>>> start_samples: 0 + +INFO:megatron.core.datasets.gpt_dataset: > Found complemental loss_mask dataset. Loading from: /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_loss_mask_tmp +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +>>> start_samples: 0 +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + + + +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDataset + + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 2031 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 2031 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 15046 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 15046 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 3299 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 3299 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 9228 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 9228 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 166555 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 166555 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 7133 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 7133 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 25714 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 25714 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 86039 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 86039 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 11193 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 11193 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 63250 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 63250 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 36116 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 36116 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 527 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 527 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 59344 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 59344 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 294 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 294 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 484 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 484 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 122883 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 122883 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 163 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 163 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 63156 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 63156 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 778640 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 778640 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 43604 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 43604 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 175670 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 175670 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 226991 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 226991 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 8162 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 8162 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 12434 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 12434 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 28587 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 28587 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 199602 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 199602 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 2784740 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 2784740 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1829337 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1829337 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 178682 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 178682 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 52867 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 52867 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 498 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 498 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 177418 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 177418 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 519 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 519 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 78 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 78 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 84548 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 84548 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 16160 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 16160 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 93319 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 93319 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 12473 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 12473 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 228187 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 228187 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 68576 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 68576 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 189 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 189 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1647592 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1647592 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 89874 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 89874 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 373365 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 373365 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 604 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 604 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 86064 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 86064 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 17995 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 17995 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 151034 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 151034 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1849 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1849 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 148 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 148 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 35370 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 35370 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 274 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 274 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 134522 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 134522 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 79269 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 79269 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 432 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 432 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 145607 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 145607 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 579 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 579 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 31211 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 31211 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 287047 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 287047 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 130201 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 130201 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 70545 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 70545 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 104516 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 104516 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 52569 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 52569 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 413 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 413 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 39658 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 39658 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 19381 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 19381 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 263497 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 263497 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 19431 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 19431 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 69929 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 69929 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 24205 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 24205 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 142217 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 142217 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 70554 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 70554 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 260245 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 260245 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 16637 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 16637 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 10723 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 10723 +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +INFO:megatron.core.datasets.gpt_dataset:{ + "class": "GPTDataset", + "dataset_path": "/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp", + "num_samples": 320, + "index_split": "valid", + "random_seed": 1234, + "sequence_length": 524288, + "split": "999,1,0", + "split_matrix": [ + [ + 0, + 0.999 + ], + [ + 0.999, + 1.0 + ], + null + ], + "tokenizer": { + "class": "megatron.core.tokenizers.text.models.default_tokenizer.DefaultTokenizerText", + "tokenizer_path": "/mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp", + "vocab_file": "None", + "merges_file": "None" + } +} +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset valid indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random-sample_index.npy +pad_document_marker: 12051293 +pad_document_marker: 12051293/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 3495 +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +> finished creating GPT datasets ... GPTDataset +> Building pretraining data loader with total train samples: 145599, consumed train samples: 1280 +>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + + + + +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDataset + + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +[after dataloaders are built] datetime: 2026-10-11 16:14:36 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 40... +[before the start of training step] datetime: 2026-10-11 16:14:36 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + + + + + + +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +WARNING:megatron.core.rerun_state_machine:Result validation enabled +Theoretical memory footprints not yet supported for hybrid Mamba-Transformer models. +[Rank 7] (after 41 iterations) memory (MB) | allocated: 47155.41455078125 | max allocated: 51321.1875 | reserved: 54360.0 | max reserved: 54360.0 | device usage: 62385.3125[Rank 1] (after 41 iterations) memory (MB) | allocated: 47155.4150390625 | max allocated: 51373.6953125 | reserved: 54194.0 | max reserved: 54194.0 | device usage: 62427.3125 +[Rank 5] (after 41 iterations) memory (MB) | allocated: 47155.4111328125 | max allocated: 51459.6640625 | reserved: 54478.0 | max reserved: 54478.0 | device usage: 62711.3125 + +[Rank 3] (after 41 iterations) memory (MB) | allocated: 47155.412109375 | max allocated: 51306.23486328125 | reserved: 54104.0 | max reserved: 54104.0 | device usage: 62337.3125[Rank 2] (after 41 iterations) memory (MB) | allocated: 47155.408203125 | max allocated: 51475.75634765625 | reserved: 53938.0 | max reserved: 53938.0 | device usage: 62171.3125 +[Rank 4] (after 41 iterations) memory (MB) | allocated: 47155.40869140625 | max allocated: 51538.62451171875 | reserved: 54278.0 | max reserved: 54278.0 | device usage: 62511.3125 + +[Rank 0] (after 41 iterations) memory (MB) | allocated: 47155.40966796875 | max allocated: 51293.7978515625 | reserved: 53670.0 | max reserved: 53670.0 | device usage: 61801.3125[Rank 6] (after 41 iterations) memory (MB) | allocated: 47155.4150390625 | max allocated: 51383.212890625 | reserved: 54042.0 | max reserved: 54042.0 | device usage: 62275.3125 + +saving checkpoint at iteration 80 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable +W1011 18:47:44.418000 937335 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 18:47:44.418000 937335 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( + scheduled an async checkpoint save at iteration 80 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 80 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 40 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 80 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 120 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 160 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 200 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 240 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 280 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 320 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 360 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 400 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 440 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 480 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 520 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 560 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 600 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 640 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 680 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 720 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 760 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 800 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 840 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 880 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 920 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +Iter 1000 increase timing-log-level to 2. + successfully saved checkpoint from iteration 1000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 960 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +Iter 1005 reset timing-log-level. checkpoint path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints; tensorboard path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard +saving checkpoint at iteration 1040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1000 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1040 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1080 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1120 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1160 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1200 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1240 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1280 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1320 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1360 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1400 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1440 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1480 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1520 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1560 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1600 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1640 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1680 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1720 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1760 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1800 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1840 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1880 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 1960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 1960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1920 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 2000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +Iter 2000 increase timing-log-level to 2. + successfully saved checkpoint from iteration 2000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1960 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +Iter 2005 reset timing-log-level. checkpoint path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints; tensorboard path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard +saving checkpoint at iteration 2040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 2040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2000 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 2080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 2080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2040 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 2120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 2120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2080 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 2160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 2160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2120 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 2200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 2200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2160 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 2240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 2240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2200 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 2280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 2280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2240 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 2320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 2320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2280 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +saving checkpoint at iteration 2360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints + successfully saved checkpoint from iteration 2360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2320 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +[after training is done] datetime: 2026-10-17 19:38:54 +saving checkpoint at iteration 2384 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2384 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +WARNING:megatron.core.rerun_state_machine:Setting RerunStateMachine mode disabled +Evaluating on 320 samples +Evaluating iter 1/10 +Evaluating iter 2/10 +Evaluating iter 3/10 +Evaluating iter 4/10 +Evaluating iter 5/10 +Evaluating iter 6/10 +Evaluating iter 7/10 +Evaluating iter 8/10 +Evaluating iter 9/10 +Evaluating iter 10/10 +WARNING:megatron.core.rerun_state_machine:Setting RerunStateMachine mode validate_results +WARNING:megatron.core.rerun_state_machine:Setting RerunStateMachine mode validate_results +Unfinalized async checkpoint saves. Finalizing them synchronously now. + successfully saved checkpoint from iteration 2384 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2360 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints +[rank1]:[W1017 19:49:59.600338206 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank0]:[W1017 19:49:59.617115255 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W1017 19:49:59.739530552 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W1017 19:50:00.768070779 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W1017 19:50:00.791559412 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W1017 19:50:00.830081801 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W1017 19:50:00.857631889 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W1017 19:50:00.932302244 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node0.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node0.sh new file mode 100644 index 0000000000000000000000000000000000000000..b8d06179a17bb5a7648ed6645af946127aeba191 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node0.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Stage16a (two-phase plan, run A): 40B tokens normal attention on part1_40B split. +# 512K seq length, loads stage15 final weights (resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 32 docs x 524288 = 16,777,216 tokens/step; 76288 samples = 2384 iters. +# part1 = 39.9998B tokens (every source cut at the same doc ratio, see split_20261010/split_report.json). +# RESUME from iter_0000040 with CP32 (16K tokens/rank = stage15-proven memory profile). continual-pretrain DROPPED so iteration/optimizer/scheduler/RNG resume from the ckpt. +# After completion, stage16b resumes from this run's final ckpt with sparse attention. +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 0 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 32 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node1.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node1.log new file mode 100644 index 0000000000000000000000000000000000000000..b804f855fa0fd51757be335b4374173bee416b88 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node1.log @@ -0,0 +1,382 @@ +W1011 16:05:33.420000 51141 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 16:05:33.420000 51141 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +W1011 16:05:33.446000 51142 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 16:05:33.446000 51142 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank 11 is connected to 63[Gloo] Rank peer ranks. Expected number of connected peer ranks is : 6313 is connected to +63 peer ranks. Expected number of connected peer ranks is : 63 +[rank11]:[W1011 16:06:36.781916462 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank13]:[W1011 16:06:36.781916635 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank11]:[W1011 16:06:36.781928218 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank13]:[W1011 16:06:36.781930159 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 14 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank14]:[W1011 16:06:36.782415923 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank14]:[W1011 16:06:36.782426634 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 8 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 10 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank8]:[W1011 16:06:36.934827938 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank8]:[W1011 16:06:36.934836869 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank10]:[W1011 16:06:36.934843883 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank10]:[W1011 16:06:36.934853404 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 9 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 15 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 12 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank9]:[W1011 16:06:36.935147461 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank9]:[W1011 16:06:36.935160644 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank15]:[W1011 16:06:36.935209444 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank15]:[W1011 16:06:36.935220449 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank12]:[W1011 16:06:36.935299541 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank12]:[W1011 16:06:36.935313004 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank12]:[W1011 16:06:37.051335544 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 12] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank11]:[W1011 16:06:37.051343753 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 11] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank13]:[W1011 16:06:37.051343482 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 13] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank14]:[W1011 16:06:37.051347669 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 14] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank10]:[W1011 16:06:37.051718972 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 10] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank8]:[W1011 16:06:37.052664219 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 8] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank9]:[W1011 16:06:37.055627906 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 9] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank15]:[W1011 16:06:37.056839384 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 15] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank14]:[W1011 16:06:38.331721687 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank14]:[W1011 16:06:38.331746309 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank13]:[W1011 16:06:38.331823461 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank13]:[W1011 16:06:38.331846050 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank12]:[W1011 16:06:38.331837301 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank12]:[W1011 16:06:38.331906824 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank9]:[W1011 16:06:38.331928923 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank15]:[W1011 16:06:38.331939636 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank9]:[W1011 16:06:38.331953099 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank15]:[W1011 16:06:38.331954820 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank8]:[W1011 16:06:38.332058259 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank8]:[W1011 16:06:38.332073154 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank10]:[W1011 16:06:38.332075580 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank10]:[W1011 16:06:38.332092607 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank11]:[W1011 16:06:38.332115493 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank11]:[W1011 16:06:38.332130582 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 + + + + +>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDataset + + +Initializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDataset + + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + +pad_document_marker: 12051293 +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293pad_document_marker: 12051293 + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: Truepad_document_marker: 12051293/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True + + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: Truepad_document_marker: 12051293pad_document_marker: 12051293 + + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + + + +>>> start_samples: 0 +>>> start_samples: 0 + +>>> start_samples: 0 +Initializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + + + + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + + + + + + +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +[Rank 9] (after 41 iterations) memory (MB) | allocated: 47155.4228515625 | max allocated: 51367.4248046875 | reserved: 54384.0 | max reserved: 54384.0 | device usage: 62745.6875[Rank 15] (after 41 iterations) memory (MB) | allocated: 47155.41455078125 | max allocated: 51329.705078125 | reserved: 54448.0 | max reserved: 54448.0 | device usage: 62601.6875[Rank 10] (after 41 iterations) memory (MB) | allocated: 47155.41015625 | max allocated: 51466.95703125 | reserved: 54418.0 | max reserved: 54418.0 | device usage: 62799.6875[Rank 11] (after 41 iterations) memory (MB) | allocated: 47155.4130859375 | max allocated: 51319.537109375 | reserved: 54318.0 | max reserved: 54318.0 | device usage: 62679.6875 + + + +[Rank 12] (after 41 iterations) memory (MB) | allocated: 47155.41064453125 | max allocated: 51518.2685546875 | reserved: 54584.0 | max reserved: 54584.0 | device usage: 62965.6875 +[Rank 13] (after 41 iterations) memory (MB) | allocated: 47155.4140625 | max allocated: 51455.5068359375 | reserved: 54220.0 | max reserved: 54220.0 | device usage: 62581.6875 +[Rank 8] (after 41 iterations) memory (MB) | allocated: 47155.40966796875 | max allocated: 51288.97998046875 | reserved: 54100.0 | max reserved: 54100.0 | device usage: 62529.6875 +[Rank 14] (after 41 iterations) memory (MB) | allocated: 47155.41259765625 | max allocated: 51390.32421875 | reserved: 54206.0 | max reserved: 54206.0 | device usage: 62587.6875 +W1011 18:47:36.216000 12216 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 18:47:36.216000 12216 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank11]:[W1017 19:49:59.291737965 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank13]:[W1017 19:49:59.358122687 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank12]:[W1017 19:49:59.412380567 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank10]:[W1017 19:49:59.466646703 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank15]:[W1017 19:49:59.474400783 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank14]:[W1017 19:49:59.487229471 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank8]:[W1017 19:49:59.487719686 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank9]:[W1017 19:49:59.540714753 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node1.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node1.sh new file mode 100644 index 0000000000000000000000000000000000000000..bb4dfef86880659796467b466bfbe0185b7f884b --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node1.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 1 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 32 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node2.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node2.log new file mode 100644 index 0000000000000000000000000000000000000000..c6963c04b0a628d653f0f6a028fec02148186e09 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node2.log @@ -0,0 +1,380 @@ +W1011 16:05:38.161000 1034365 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 16:05:38.161000 1034365 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank 19 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank19]:[W1011 16:06:36.347971609 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank19]:[W1011 16:06:36.347983217 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank [Gloo] Rank 22 is connected to 6317 peer ranks. is connected to Expected number of connected peer ranks is : 6363 peer ranks. Expected number of connected peer ranks is : +63 +[rank22]:[W1011 16:06:36.352198384 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank17]:[W1011 16:06:36.352198842 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank22]:[W1011 16:06:36.352209599 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank17]:[W1011 16:06:36.352210821 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 16 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank16]:[W1011 16:06:36.352840812 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank16]:[W1011 16:06:36.352850429 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank [Gloo] Rank 20 is connected to 2163 is connected to peer ranks. 63Expected number of connected peer ranks is : peer ranks. 63Expected number of connected peer ranks is : +63 +[Gloo] Rank 23 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 18 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank21]:[W1011 16:06:36.353000655 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank21]:[W1011 16:06:36.353010629 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank20]:[W1011 16:06:36.353009455 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank20]:[W1011 16:06:36.353019741 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank23]:[W1011 16:06:36.353040290 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank23]:[W1011 16:06:36.353081509 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank18]:[W1011 16:06:36.353093542 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank18]:[W1011 16:06:36.353103122 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1[Gloo] Rank +0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank16]:[W1011 16:06:37.648287911 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 16] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank23]:[W1011 16:06:37.648293807 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 23] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank17]:[W1011 16:06:37.648299149 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 17] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank20]:[W1011 16:06:37.648303876 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 20] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank18]:[W1011 16:06:37.648307236 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 18] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank21]:[W1011 16:06:37.648306145 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 21] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank22]:[W1011 16:06:37.648313138 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 22] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank19]:[W1011 16:06:37.649018689 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 19] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank19]:[W1011 16:06:38.920531463 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank19]:[W1011 16:06:38.920553875 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank23]:[W1011 16:06:38.920642576 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank20]:[W1011 16:06:38.920649349 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank23]:[W1011 16:06:38.920660469 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank20]:[W1011 16:06:38.920664055 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank21]:[W1011 16:06:38.920668325 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank18]:[W1011 16:06:38.920707232 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank21]:[W1011 16:06:38.920727609 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank18]:[W1011 16:06:38.920729319 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank17]:[W1011 16:06:38.920746365 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank17]:[W1011 16:06:38.920767923 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank16]:[W1011 16:06:38.920971442 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank16]:[W1011 16:06:38.920987750 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank22]:[W1011 16:06:38.921035086 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank22]:[W1011 16:06:38.921054872 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + + +>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset>>> start_samples: 0 + +>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +pad_document_marker: 12051293 +pad_document_marker: 12051293 +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True + +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + + + + + + +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset>>> start_samples: 0 + +Initializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDataset + + + +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + + + + +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +[Rank 16] (after 41 iterations) memory (MB) | allocated: 47155.4150390625 | max allocated: 51303.95361328125 | reserved: 54228.0 | max reserved: 54228.0 | device usage: 62637.625[Rank 22] (after 41 iterations) memory (MB) | allocated: 47155.4130859375 | max allocated: 51403.38037109375 | reserved: 54436.0 | max reserved: 54436.0 | device usage: 62797.625[Rank 21] (after 41 iterations) memory (MB) | allocated: 47155.41015625 | max allocated: 51463.2626953125 | reserved: 54378.0 | max reserved: 54378.0 | device usage: 62759.625 +[Rank 18] (after 41 iterations) memory (MB) | allocated: 47155.40966796875 | max allocated: 51479.5986328125 | reserved: 54560.0 | max reserved: 54560.0 | device usage: 62921.625 +[Rank 19] (after 41 iterations) memory (MB) | allocated: 47155.4091796875 | max allocated: 51327.64892578125 | reserved: 54178.0 | max reserved: 54178.0 | device usage: 62559.625 + + +[Rank 20] (after 41 iterations) memory (MB) | allocated: 47155.4140625 | max allocated: 51482.6552734375 | reserved: 54504.0 | max reserved: 54504.0 | device usage: 62865.625 +[Rank 23] (after 41 iterations) memory (MB) | allocated: 47155.41064453125 | max allocated: 51338.990234375 | reserved: 54260.0 | max reserved: 54260.0 | device usage: 62433.625 +[Rank 17] (after 41 iterations) memory (MB) | allocated: 47155.4140625 | max allocated: 51380.9580078125 | reserved: 54462.0 | max reserved: 54462.0 | device usage: 62843.625 +W1011 18:47:44.233000 188191 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 18:47:44.233000 188191 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank21]:[W1017 19:49:59.057805559 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank20]:[W1017 19:49:59.091821948 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank22]:[W1017 19:49:59.110002545 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank19]:[W1017 19:49:59.138910274 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank18]:[W1017 19:50:00.182297982 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank23]:[W1017 19:50:00.190838408 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank16]:[W1017 19:50:00.263683259 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank17]:[W1017 19:50:00.617903953 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node2.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node2.sh new file mode 100644 index 0000000000000000000000000000000000000000..41bc2553f670a8d1bee9396a58144fb9c89cf979 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node2.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 2 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 32 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node3.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node3.log new file mode 100644 index 0000000000000000000000000000000000000000..7b844a7008564a7c9911300320388c52c73b607a --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node3.log @@ -0,0 +1,380 @@ +W1011 16:05:50.622000 42946 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 16:05:50.622000 42946 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank 24 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63[Gloo] Rank +25 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 26 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank26]:[W1011 16:06:37.325676757 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank26]:[W1011 16:06:37.325695475 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank25]:[W1011 16:06:37.325706720 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank24]:[W1011 16:06:37.325710491 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank25]:[W1011 16:06:37.325731444 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank24]:[W1011 16:06:37.325737032 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 27 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank27]:[W1011 16:06:37.326283974 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank27]:[W1011 16:06:37.326302622 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 28 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank28]:[W1011 16:06:37.327098626 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank28]:[W1011 16:06:37.327119509 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 29 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank29]:[W1011 16:06:37.327539336 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank29]:[W1011 16:06:37.327557845 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 30 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 31 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank31]:[W1011 16:06:37.328698524 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank31]:[W1011 16:06:37.328827804 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank30]:[W1011 16:06:37.329983640 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank30]:[W1011 16:06:37.330010739 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 0 is connected to 1 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : [Gloo] Rank 0 is connected to 110[Gloo] Rank [Gloo] Rank peer ranks. + is connected to 00Expected number of connected peer ranks is : 1[Gloo] Rank is connected to is connected to 1 peer ranks. 011[Gloo] Rank +Expected number of connected peer ranks is : is connected to peer ranks. peer ranks. 01[Gloo] Rank 1Expected number of connected peer ranks is : Expected number of connected peer ranks is : is connected to +0 peer ranks. 111 is connected to Expected number of connected peer ranks is : + + peer ranks. 11Expected number of connected peer ranks is : peer ranks. +1Expected number of connected peer ranks is : +1 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7[Gloo] Rank peer ranks. 3Expected number of connected peer ranks is : is connected to 77 + peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank29]:[W1011 16:06:37.426196172 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 29] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank27]:[W1011 16:06:37.426210559 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 27] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank24]:[W1011 16:06:37.426226204 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 24] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank28]:[W1011 16:06:37.426249532 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 28] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank31]:[W1011 16:06:37.426273977 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 31] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank25]:[W1011 16:06:37.426288644 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 25] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank26]:[W1011 16:06:37.426305965 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 26] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank30]:[W1011 16:06:37.426332925 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 30] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank28]:[W1011 16:06:38.736245016 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank28]:[W1011 16:06:38.736278402 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank30]:[W1011 16:06:38.736549952 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank30]:[W1011 16:06:38.736591509 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank31]:[W1011 16:06:38.736603871 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank31]:[W1011 16:06:38.736629643 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank24]:[W1011 16:06:38.737843937 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank24]:[W1011 16:06:38.738003110 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank27]:[W1011 16:06:38.738057099 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank27]:[W1011 16:06:38.738080496 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank26]:[W1011 16:06:38.738394791 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank26]:[W1011 16:06:38.738414277 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank29]:[W1011 16:06:38.738566675 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank29]:[W1011 16:06:38.738594822 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank25]:[W1011 16:06:38.738655865 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank25]:[W1011 16:06:38.738683243 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0>>> start_samples: 0 + + + +>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDataset + + + +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + +pad_document_marker: 12051293pad_document_marker: 12051293 + +pad_document_marker: 12051293 +pad_document_marker: 12051293 +pad_document_marker: 12051293 +pad_document_marker: 12051293 +pad_document_marker: 12051293 +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True + +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 + + + +>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDataset>>> start_samples: 0Initializing GlobIndexedDataset + + + + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + + + + +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +[Rank 25] (after 41 iterations) memory (MB) | allocated: 47155.4130859375 | max allocated: 51363.109375 | reserved: 54818.0 | max reserved: 54818.0 | device usage: 63199.75[Rank 31] (after 41 iterations) memory (MB) | allocated: 47155.41064453125 | max allocated: 51342.35546875 | reserved: 54518.0 | max reserved: 54518.0 | device usage: 62691.75[Rank 27] (after 41 iterations) memory (MB) | allocated: 47155.408203125 | max allocated: 51322.27490234375 | reserved: 54642.0 | max reserved: 54642.0 | device usage: 63023.75 +[Rank 30] (after 41 iterations) memory (MB) | allocated: 47155.41015625 | max allocated: 51361.72119140625 | reserved: 54400.0 | max reserved: 54400.0 | device usage: 62781.75 + +[Rank 29] (after 41 iterations) memory (MB) | allocated: 47155.4111328125 | max allocated: 51472.71875 | reserved: 54582.0 | max reserved: 54582.0 | device usage: 62963.75 +[Rank 26] (after 41 iterations) memory (MB) | allocated: 47155.4091796875 | max allocated: 51460.94140625 | reserved: 54638.0 | max reserved: 54638.0 | device usage: 63019.75 +[Rank 28] (after 41 iterations) memory (MB) | allocated: 47155.41845703125 | max allocated: 51490.8876953125 | reserved: 54404.0 | max reserved: 54404.0 | device usage: 62785.75 +[Rank 24] (after 41 iterations) memory (MB) | allocated: 47155.41162109375 | max allocated: 51298.70458984375 | reserved: 54438.0 | max reserved: 54438.0 | device usage: 62867.75 + +W1011 18:47:56.722000 31199 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 18:47:56.722000 31199 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank27]:[W1017 19:50:01.951594016 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank25]:[W1017 19:50:01.955791751 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank26]:[W1017 19:50:01.005669883 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank30]:[W1017 19:50:01.015016241 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank31]:[W1017 19:50:01.061068403 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank24]:[W1017 19:50:01.070712773 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank29]:[W1017 19:50:01.074423226 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank28]:[W1017 19:50:01.096244966 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node3.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node3.sh new file mode 100644 index 0000000000000000000000000000000000000000..669b4d882bd22aab2209d115e8792170d8dcd5fb --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node3.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 3 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 32 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node4.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node4.log new file mode 100644 index 0000000000000000000000000000000000000000..6feff8164861f815a98d063a766e26713a50d386 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node4.log @@ -0,0 +1,365 @@ +W1011 16:05:40.894000 33364 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 16:05:40.894000 33364 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 3734323533363839 is connected to is connected to is connected to is connected to is connected to is connected to is connected to is connected to 6363636363636363 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 636363 + +6363636363 + + + + + +[rank35]:[W1011 16:06:37.171295400 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank36]:[W1011 16:06:37.171301702 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank35]:[W1011 16:06:37.171310970 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank33]:[W1011 16:06:37.171309718 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank36]:[W1011 16:06:37.171317553 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank39]:[W1011 16:06:37.171316330 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank33]:[W1011 16:06:37.171324516 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank38]:[W1011 16:06:37.171322892 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank39]:[W1011 16:06:37.171332601 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank37]:[W1011 16:06:37.171326960 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank38]:[W1011 16:06:37.171335667 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank34]:[W1011 16:06:37.171337190 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank32]:[W1011 16:06:37.171339564 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank37]:[W1011 16:06:37.171346187 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank34]:[W1011 16:06:37.171352328 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank32]:[W1011 16:06:37.171384650 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank [Gloo] Rank 11 is connected to 1 is connected to peer ranks. 1Expected number of connected peer ranks is : peer ranks. 1Expected number of connected peer ranks is : +1[Gloo] Rank +1[Gloo] Rank [Gloo] Rank is connected to 111 is connected to is connected to peer ranks. 11Expected number of connected peer ranks is : peer ranks. peer ranks. 1Expected number of connected peer ranks is : Expected number of connected peer ranks is : +11 + +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank34]:[W1011 16:06:37.253129656 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 34] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank32]:[W1011 16:06:37.253129556 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 32] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank35]:[W1011 16:06:37.253129756 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 35] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank33]:[W1011 16:06:37.253139585 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 33] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank39]:[W1011 16:06:37.253140086 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 39] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank36]:[W1011 16:06:37.253147600 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 36] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank38]:[W1011 16:06:37.253145897 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 38] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank37]:[W1011 16:06:37.253146618 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 37] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank39]:[W1011 16:06:38.555417649 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank35]:[W1011 16:06:38.555417629 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank39]:[W1011 16:06:38.555450561 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank35]:[W1011 16:06:38.555454389 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank38]:[W1011 16:06:38.555588063 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank38]:[W1011 16:06:38.555602991 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank33]:[W1011 16:06:38.555650081 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank33]:[W1011 16:06:38.555723851 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank32]:[W1011 16:06:38.556477403 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank34]:[W1011 16:06:38.556503893 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank32]:[W1011 16:06:38.556513331 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank34]:[W1011 16:06:38.556522749 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank37]:[W1011 16:06:38.556588574 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank37]:[W1011 16:06:38.556626486 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank36]:[W1011 16:06:38.556696730 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank36]:[W1011 16:06:38.556730564 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0Initializing GlobIndexedDataset + +>>> start_samples: 0 + +Initializing GlobIndexedDataset + + +>>> start_samples: 0 +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +pad_document_marker: 12051293 +pad_document_marker: 12051293 +pad_document_marker: 12051293 +pad_document_marker: 12051293 +pad_document_marker: 12051293 +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +pad_document_marker: 12051293/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + + +>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +>>> start_samples: 0 +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + + + + + +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +W1011 18:47:46.785000 52238 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 18:47:46.785000 52238 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank32]:[W1017 19:50:00.276358316 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node4.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node4.sh new file mode 100644 index 0000000000000000000000000000000000000000..581e8f213af00876250b8dae575a0d514290ab81 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node4.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 4 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 32 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node5.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node5.log new file mode 100644 index 0000000000000000000000000000000000000000..f6fefcab87135ef3ae105c588610ae8cc6d50896 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node5.log @@ -0,0 +1,365 @@ +W1011 16:05:37.947000 10642 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 16:05:37.947000 10642 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank 43 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank43]:[W1011 16:06:37.869437502 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank43]:[W1011 16:06:37.869449300 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 40 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank40]:[W1011 16:06:37.871851282 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank40]:[W1011 16:06:37.871859863 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 41 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank [Gloo] Rank 45 is connected to 63 peer ranks. 42Expected number of connected peer ranks is : is connected to 6363 + peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 46 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank41]:[W1011 16:06:37.872141438 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank41]:[W1011 16:06:37.872151954 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 47 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank45]:[W1011 16:06:37.872164994 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank45]:[W1011 16:06:37.872175812 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank42]:[W1011 16:06:37.872188535 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank42]:[W1011 16:06:37.872199373 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank46]:[W1011 16:06:37.872200404 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank46]:[W1011 16:06:37.872209840 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank47]:[W1011 16:06:37.872267334 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank47]:[W1011 16:06:37.872280016 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 44 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank44]:[W1011 16:06:37.872758291 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank44]:[W1011 16:06:37.872767745 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank [Gloo] Rank 11 is connected to is connected to 11 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 11 + +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank46]:[W1011 16:06:37.952305508 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 46] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank43]:[W1011 16:06:37.952313957 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 43] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank41]:[W1011 16:06:37.952314067 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 41] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank40]:[W1011 16:06:37.952314801 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 40] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank45]:[W1011 16:06:37.952318647 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 45] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank42]:[W1011 16:06:37.952325370 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 42] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank44]:[W1011 16:06:37.952329533 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 44] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank47]:[W1011 16:06:37.952727226 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 47] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank43]:[W1011 16:06:38.225355092 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank43]:[W1011 16:06:38.225378188 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank47]:[W1011 16:06:38.225716848 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank47]:[W1011 16:06:38.225732278 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank42]:[W1011 16:06:38.225733305 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank42]:[W1011 16:06:38.225747958 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank46]:[W1011 16:06:38.225749294 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank45]:[W1011 16:06:38.225769548 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank44]:[W1011 16:06:38.225786135 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank46]:[W1011 16:06:38.225806054 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank45]:[W1011 16:06:38.225808580 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank44]:[W1011 16:06:38.225813794 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank41]:[W1011 16:06:38.225820007 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank41]:[W1011 16:06:38.225841120 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank40]:[W1011 16:06:38.225859750 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank40]:[W1011 16:06:38.225875158 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + + + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0 +Initializing GlobIndexedDataset>>> start_samples: 0Initializing GlobIndexedDataset + + +>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDataset +>>> start_samples: 0>>> start_samples: 0 +Initializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +pad_document_marker: 12051293 +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True + +pad_document_marker: 12051293 +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0 + + +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset +Initializing GlobIndexedDataset + +>>> start_samples: 0 +Initializing GlobIndexedDataset +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + + + + + + +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +W1011 18:47:43.733000 30139 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 18:47:43.733000 30139 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank40]:[W1017 19:49:59.512524191 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node5.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node5.sh new file mode 100644 index 0000000000000000000000000000000000000000..1d4f6483ff96060580f065e234b95c6a3c7bab35 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node5.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 5 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 32 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node6.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node6.log new file mode 100644 index 0000000000000000000000000000000000000000..7bdcfc06dfe4017633ad8e25dfdf4d7b5c8b3567 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node6.log @@ -0,0 +1,365 @@ +W1011 16:05:40.654000 27228 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 16:05:40.654000 27228 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank [Gloo] Rank 4849 is connected to is connected to 6363 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 6363 + +[Gloo] Rank 50 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank48]:[W1011 16:06:37.473180093 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank48]:[W1011 16:06:37.473198367 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank49]:[W1011 16:06:37.473193328 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank49]:[W1011 16:06:37.473212214 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank50]:[W1011 16:06:37.473225278 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank50]:[W1011 16:06:37.473236820 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 51 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank51]:[W1011 16:06:37.474129975 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank51]:[W1011 16:06:37.474148981 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 52 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank52]:[W1011 16:06:37.474654249 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank52]:[W1011 16:06:37.474663908 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 53 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank53]:[W1011 16:06:37.475251783 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank53]:[W1011 16:06:37.475263325 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 54 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 55 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank54]:[W1011 16:06:37.475790174 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank54]:[W1011 16:06:37.475857261 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank55]:[W1011 16:06:37.475954646 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank55]:[W1011 16:06:37.475971628 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank [Gloo] Rank 11 is connected to is connected to 11 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 11 + +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank48]:[W1011 16:06:37.561393374 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 48] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank52]:[W1011 16:06:37.561400267 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 52] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank54]:[W1011 16:06:37.561406128 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 54] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank49]:[W1011 16:06:37.561405567 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 49] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank51]:[W1011 16:06:37.561945481 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 51] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank50]:[W1011 16:06:37.562411535 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 50] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank53]:[W1011 16:06:37.562753454 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 53] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank55]:[W1011 16:06:37.563081286 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 55] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank48]:[W1011 16:06:38.858620659 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank48]:[W1011 16:06:38.858645636 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank52]:[W1011 16:06:38.859242920 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank52]:[W1011 16:06:38.859276063 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank50]:[W1011 16:06:38.859297083 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank50]:[W1011 16:06:38.859341917 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank55]:[W1011 16:06:38.859376483 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank55]:[W1011 16:06:38.859501099 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank54]:[W1011 16:06:38.859700277 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank54]:[W1011 16:06:38.859730955 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank49]:[W1011 16:06:38.859774117 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank49]:[W1011 16:06:38.859792102 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank51]:[W1011 16:06:38.860027789 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank51]:[W1011 16:06:38.860053848 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank53]:[W1011 16:06:38.864390514 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank53]:[W1011 16:06:38.864408889 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + + + +>>> start_samples: 0 +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + + +>>> start_samples: 0 + +>>> start_samples: 0Initializing GlobIndexedDataset +>>> start_samples: 0 + +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + +pad_document_marker: 12051293 +pad_document_marker: 12051293 +pad_document_marker: 12051293 +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: Truepad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True + + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +pad_document_marker: 12051293 +pad_document_marker: 12051293/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True + +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True + +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 + +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + + + + +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +W1011 18:47:45.266000 56560 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 18:47:45.266000 56560 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank48]:[W1017 19:50:00.424604584 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node6.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node6.sh new file mode 100644 index 0000000000000000000000000000000000000000..ceccc7002968407bc4e7817f964065cc2f9e0f8b --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node6.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 6 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 32 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node7.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node7.log new file mode 100644 index 0000000000000000000000000000000000000000..9123feb7e4f9658913a99edd924f864947a689d1 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node7.log @@ -0,0 +1,14665 @@ +W1011 16:05:32.575000 344174 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 16:05:32.575000 344174 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +WARNING: TensorBoard writing requested but is not available (are you using PyTorch 1.1.0 or later?), no TensorBoard logs will be written. +wandb: Currently logged in as: huyiwen to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run acr8mx9p +wandb: Tracking run with wandb version 0.23.0 +wandb: Run data is saved locally in /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_160551-acr8mx9p +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run stage15_115B_8node_gdnlralpha_off_20261003a +wandb: ⭐️ View project at https://wandb.ai/huyiwen/qwen2.5_3b_mom_cpt +wandb: 🚀 View run at https://wandb.ai/huyiwen/qwen2.5_3b_mom_cpt/runs/acr8mx9p +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 56[Gloo] Rank is connected to 63 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 5863[Gloo] Rank 60 is connected to + is connected to [Gloo] Rank 576363 is connected to peer ranks. [Gloo] Rank 62 peer ranks. 63[Gloo] Rank Expected number of connected peer ranks is : is connected to [Gloo] Rank 59Expected number of connected peer ranks is : peer ranks. 61636363 is connected to 63Expected number of connected peer ranks is : is connected to + peer ranks. is connected to 63 +6363Expected number of connected peer ranks is : 63 peer ranks. + peer ranks. 63 peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : +Expected number of connected peer ranks is : 636363 + + +[rank63]:[W1011 16:06:37.977089079 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank61]:[W1011 16:06:37.977098568 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank59]:[W1011 16:06:37.977096704 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank63]:[W1011 16:06:37.977102139 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank58]:[W1011 16:06:37.977096649 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank60]:[W1011 16:06:37.977096684 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank62]:[W1011 16:06:37.977101893 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank61]:[W1011 16:06:37.977109545 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank57]:[W1011 16:06:37.977105775 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank59]:[W1011 16:06:37.977112426 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank58]:[W1011 16:06:37.977116065 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank60]:[W1011 16:06:37.977118365 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank62]:[W1011 16:06:37.977120505 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank56]:[W1011 16:06:37.977113546 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank57]:[W1011 16:06:37.977150177 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank56]:[W1011 16:06:37.977162724 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 1 is connected to 1 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 1[Gloo] Rank 1 is connected to [Gloo] Rank [Gloo] Rank 1 +111 is connected to [Gloo] Rank peer ranks. is connected to is connected to 11Expected number of connected peer ranks is : 1 peer ranks. is connected to 1 peer ranks. 1Expected number of connected peer ranks is : 1 +Expected number of connected peer ranks is : peer ranks. 1 peer ranks. 1Expected number of connected peer ranks is : +Expected number of connected peer ranks is : +11 + +[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank57]:[W1011 16:06:37.040330930 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 57] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank62]:[W1011 16:06:37.040333607 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 62] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank58]:[W1011 16:06:37.040339033 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 58] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank56]:[W1011 16:06:37.040341707 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 56] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank61]:[W1011 16:06:37.040346389 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 61] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank59]:[W1011 16:06:37.040342536 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 59] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank63]:[W1011 16:06:37.040345553 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 63] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank60]:[W1011 16:06:37.040421207 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 60] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank61]:[W1011 16:06:38.314733022 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank61]:[W1011 16:06:38.314755742 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank58]:[W1011 16:06:38.314821800 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank58]:[W1011 16:06:38.314843015 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank62]:[W1011 16:06:38.315002245 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank62]:[W1011 16:06:38.315019426 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank56]:[W1011 16:06:38.315043521 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank56]:[W1011 16:06:38.315058947 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank60]:[W1011 16:06:38.315073675 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank60]:[W1011 16:06:38.315094839 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank59]:[W1011 16:06:38.315105496 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank59]:[W1011 16:06:38.315121299 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank63]:[W1011 16:06:38.315137786 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank63]:[W1011 16:06:38.315151961 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank57]:[W1011 16:06:38.316189288 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank57]:[W1011 16:06:38.316206236 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +times across ranks (ms): + load-checkpoint: + rank 0: 107363.73 + rank 1: 107363.66 + rank 2: 107363.73 + rank 3: 107363.94 + rank 4: 107363.67 + rank 5: 107363.82 + rank 6: 107363.65 + rank 7: 107363.67 + rank 8: 107363.52 + rank 9: 107363.52 + rank 10: 107363.68 + rank 11: 107363.62 + rank 12: 107363.53 + rank 13: 107363.49 + rank 14: 107363.53 + rank 15: 107363.47 + rank 16: 107363.80 + rank 17: 107363.73 + rank 18: 107363.80 + rank 19: 107363.84 + rank 20: 107363.67 + rank 21: 107363.92 + rank 22: 107364.05 + rank 23: 107363.88 + rank 24: 107364.27 + rank 25: 107364.05 + rank 26: 107364.22 + rank 27: 107364.56 + rank 28: 107364.23 + rank 29: 107364.26 + rank 30: 107364.14 + rank 31: 107364.10 + rank 32: 107363.66 + rank 33: 107363.95 + rank 34: 107363.64 + rank 35: 107363.93 + rank 36: 107364.06 + rank 37: 107363.80 + rank 38: 107363.63 + rank 39: 107363.64 + rank 40: 107363.84 + rank 41: 107363.89 + rank 42: 107363.10 + rank 43: 107363.90 + rank 44: 107363.88 + rank 45: 107363.77 + rank 46: 107363.84 + rank 47: 107363.82 + rank 48: 107363.55 + rank 49: 107363.66 + rank 50: 107363.49 + rank 51: 107363.55 + rank 52: 107363.55 + rank 53: 107363.81 + rank 54: 107363.53 + rank 55: 107363.48 + rank 56: 107364.07 + rank 57: 107363.88 + rank 58: 107363.97 + rank 59: 107363.88 + rank 60: 107363.92 + rank 61: 107363.88 + rank 62: 107363.87 + rank 63: 107363.84 +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_input_ids_tmp'] +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 + +>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 + +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +pad_document_marker: 12051293 +pad_document_marker: 12051293 +pad_document_marker: 12051293pad_document_marker: 12051293 +pad_document_marker: 12051293 +pad_document_marker: 12051293 + +pad_document_marker: 12051293 +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True + +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 + + + +>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDataset + + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +>>> start_samples: 0Initializing GlobIndexedDataset + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] + + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_9_loss_mask_tmp'] +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +times across ranks (ms): + model-and-optimizer-setup: + rank 0: 133594.25 + rank 1: 133588.12 + rank 2: 133585.66 + rank 3: 133592.48 + rank 4: 133588.34 + rank 5: 133591.12 + rank 6: 133591.62 + rank 7: 133587.55 + rank 8: 133585.38 + rank 9: 133585.20 + rank 10: 133586.12 + rank 11: 133585.22 + rank 12: 133588.14 + rank 13: 133587.11 + rank 14: 133589.19 + rank 15: 133590.19 + rank 16: 133591.02 + rank 17: 133592.94 + rank 18: 133593.62 + rank 19: 133592.02 + rank 20: 133593.56 + rank 21: 133594.33 + rank 22: 133594.67 + rank 23: 133595.20 + rank 24: 133599.69 + rank 25: 133605.33 + rank 26: 133601.14 + rank 27: 133601.34 + rank 28: 133614.59 + rank 29: 133599.73 + rank 30: 133598.86 + rank 31: 133606.92 + rank 32: 133594.98 + rank 33: 133598.09 + rank 34: 133598.23 + rank 35: 133595.41 + rank 36: 133597.55 + rank 37: 133597.16 + rank 38: 133594.70 + rank 39: 133596.27 + rank 40: 133595.14 + rank 41: 133592.97 + rank 42: 133594.67 + rank 43: 133593.94 + rank 44: 133591.88 + rank 45: 133591.97 + rank 46: 133590.47 + rank 47: 133593.95 + rank 48: 133592.89 + rank 49: 133590.12 + rank 50: 133593.62 + rank 51: 133593.09 + rank 52: 133594.17 + rank 53: 133589.03 + rank 54: 133591.39 + rank 55: 133592.58 + rank 56: 133586.06 + rank 57: 133587.73 + rank 58: 133585.34 + rank 59: 133585.81 + rank 60: 133586.86 + rank 61: 133585.80 + rank 62: 133585.98 + rank 63: 133585.05 + train/valid/test-data-iterators-setup: + rank 0: 22041.88 + rank 1: 25482.36 + rank 2: 25482.49 + rank 3: 25381.98 + rank 4: 25382.04 + rank 5: 25371.71 + rank 6: 25371.76 + rank 7: 25371.91 + rank 8: 25482.21 + rank 9: 25482.11 + rank 10: 25482.09 + rank 11: 25482.12 + rank 12: 25482.31 + rank 13: 25482.14 + rank 14: 25482.14 + rank 15: 25482.26 + rank 16: 25482.48 + rank 17: 25495.04 + rank 18: 25495.08 + rank 19: 25495.08 + rank 20: 25482.49 + rank 21: 25482.28 + rank 22: 25482.37 + rank 23: 25482.21 + rank 24: 26596.63 + rank 25: 26649.74 + rank 26: 26649.72 + rank 27: 26649.69 + rank 28: 26649.92 + rank 29: 26649.71 + rank 30: 26596.54 + rank 31: 26605.73 + rank 32: 26649.84 + rank 33: 26649.83 + rank 34: 26650.61 + rank 35: 26649.81 + rank 36: 26649.79 + rank 37: 26649.81 + rank 38: 26649.95 + rank 39: 26649.79 + rank 40: 26649.96 + rank 41: 26649.87 + rank 42: 26650.04 + rank 43: 26649.88 + rank 44: 26649.96 + rank 45: 26649.68 + rank 46: 26649.87 + rank 47: 26649.71 + rank 48: 26649.87 + rank 49: 26649.86 + rank 50: 26649.82 + rank 51: 26649.88 + rank 52: 26649.72 + rank 53: 26650.77 + rank 54: 26649.83 + rank 55: 26649.85 + rank 56: 26650.07 + rank 57: 26649.82 + rank 58: 26649.92 + rank 59: 26649.81 + rank 60: 26650.04 + rank 61: 26650.07 + rank 62: 26649.93 + rank 63: 26650.07 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + + + +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 + [2026-10-11 16:20:03] iteration 41/ 2384 | consumed samples: 1312 | elapsed time per iteration (ms): 326507.2 | throughput per GPU (TFLOP/s/GPU): 15.5 | MFU 4.97% | FLOPs estimator: workload-aware | logical tokens/s: 51383.9 | valid tokens/s: 51383.9 | physical tokens/s: 51643.0 | loss tokens/s: 40349.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.411327E-01 | lm-global-supervised-tokens: 1.317437E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.743055E+01 | lm-token-buffer-mib: 2.323268E+02 | global_load_balancing_loss: 1.000853E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11233.623 | params rms: 0.060826 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 20:30:06.479095 | finish at 2026-10-20 12:50:09 + [2026-10-11 16:23:48] iteration 42/ 2384 | consumed samples: 1344 | elapsed time per iteration (ms): 225493.8 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 74401.9 | valid tokens/s: 74401.9 | physical tokens/s: 75730.1 | loss tokens/s: 51104.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.369315E-01 | lm-global-supervised-tokens: 1.152365E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.250638E-02 | lm-token-buffer-mib: 2.364373E+02 | global_load_balancing_loss: 9.989473E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11233.552 | params rms: 0.060826 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:41:46.593333 | finish at 2026-10-17 19:05:35 + [2026-10-11 16:27:32] iteration 43/ 2384 | consumed samples: 1376 | elapsed time per iteration (ms): 223354.2 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 75114.7 | valid tokens/s: 75114.7 | physical tokens/s: 75546.9 | loss tokens/s: 56547.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.432067E-01 | lm-global-supervised-tokens: 1.263020E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.027606E-02 | lm-token-buffer-mib: 2.333867E+02 | global_load_balancing_loss: 1.001140E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11233.481 | params rms: 0.060826 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:14:32.246120 | finish at 2026-10-17 17:42:04 + [2026-10-11 16:31:12] iteration 44/ 2384 | consumed samples: 1408 | elapsed time per iteration (ms): 220272.1 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 76165.7 | valid tokens/s: 76165.7 | physical tokens/s: 76784.7 | loss tokens/s: 56930.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.636720E-01 | lm-global-supervised-tokens: 1.254026E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.429243E-02 | lm-token-buffer-mib: 2.347867E+02 | global_load_balancing_loss: 1.000705E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11233.414 | params rms: 0.060825 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:10:36.797061 | finish at 2026-10-17 15:41:49 + [2026-10-11 16:35:50] iteration 45/ 2384 | consumed samples: 1440 | elapsed time per iteration (ms): 277810.0 | throughput per GPU (TFLOP/s/GPU): 17.4 | MFU 5.58% | FLOPs estimator: workload-aware | logical tokens/s: 60391.0 | valid tokens/s: 60391.0 | physical tokens/s: 62055.2 | loss tokens/s: 45216.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.028783E-01 | lm-global-supervised-tokens: 1.256153E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.588596E-02 | lm-token-buffer-mib: 2.347526E+02 | global_load_balancing_loss: 1.001366E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11233.347 | params rms: 0.060825 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 12:29:57.637825 | finish at 2026-10-19 05:05:47 + [2026-10-11 16:39:32] iteration 46/ 2384 | consumed samples: 1472 | elapsed time per iteration (ms): 222489.3 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75406.8 | valid tokens/s: 75406.8 | physical tokens/s: 76199.1 | loss tokens/s: 60014.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.189202E-01 | lm-global-supervised-tokens: 1.335262E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.551362E-02 | lm-token-buffer-mib: 2.342603E+02 | global_load_balancing_loss: 1.003794E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11233.280 | params rms: 0.060825 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:29:39.874175 | finish at 2026-10-17 17:09:12 + [2026-10-11 16:43:13] iteration 47/ 2384 | consumed samples: 1504 | elapsed time per iteration (ms): 220679.1 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 76025.2 | valid tokens/s: 76025.2 | physical tokens/s: 76465.4 | loss tokens/s: 60131.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.462784E-01 | lm-global-supervised-tokens: 1.326972E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.748896E-02 | lm-token-buffer-mib: 2.341247E+02 | global_load_balancing_loss: 1.001103E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11233.213 | params rms: 0.060824 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:15:27.030598 | finish at 2026-10-17 15:58:40 + [2026-10-11 16:46:53] iteration 48/ 2384 | consumed samples: 1536 | elapsed time per iteration (ms): 220024.6 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 76251.3 | valid tokens/s: 76251.3 | physical tokens/s: 76638.1 | loss tokens/s: 57205.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.446347E-01 | lm-global-supervised-tokens: 1.258657E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.075306E-02 | lm-token-buffer-mib: 2.324507E+02 | global_load_balancing_loss: 9.990532E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11233.148 | params rms: 0.060824 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:46:17.414429 | finish at 2026-10-17 15:33:10 + [2026-10-11 16:50:36] iteration 49/ 2384 | consumed samples: 1568 | elapsed time per iteration (ms): 223582.4 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 75038.2 | valid tokens/s: 75038.2 | physical tokens/s: 76004.3 | loss tokens/s: 56727.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.317672E-01 | lm-global-supervised-tokens: 1.268317E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.465902E-02 | lm-token-buffer-mib: 2.363987E+02 | global_load_balancing_loss: 9.998465E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11233.082 | params rms: 0.060823 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:01:05.006073 | finish at 2026-10-17 17:51:41 + [2026-10-11 16:54:37] iteration 50/ 2384 | consumed samples: 1600 | elapsed time per iteration (ms): 240576.7 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 69737.5 | valid tokens/s: 69737.5 | physical tokens/s: 70013.9 | loss tokens/s: 50210.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.235597E-01 | lm-global-supervised-tokens: 1.207950E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.998382E-02 | lm-token-buffer-mib: 2.333822E+02 | global_load_balancing_loss: 9.989101E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11233.016 | params rms: 0.060823 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:58:25.942612 | finish at 2026-10-18 04:53:03 + [2026-10-11 16:58:17] iteration 51/ 2384 | consumed samples: 1632 | elapsed time per iteration (ms): 219759.3 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 76343.4 | valid tokens/s: 76343.4 | physical tokens/s: 76706.2 | loss tokens/s: 55545.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.271363E-01 | lm-global-supervised-tokens: 1.220662E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.892759E-02 | lm-token-buffer-mib: 2.322447E+02 | global_load_balancing_loss: 1.001293E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11232.950 | params rms: 0.060823 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:24:58.362338 | finish at 2026-10-17 15:23:15 + [2026-10-11 17:02:02] iteration 52/ 2384 | consumed samples: 1664 | elapsed time per iteration (ms): 225181.7 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.83% | FLOPs estimator: workload-aware | logical tokens/s: 74505.1 | valid tokens/s: 74505.1 | physical tokens/s: 75678.5 | loss tokens/s: 56918.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.788029E-01 | lm-global-supervised-tokens: 1.281705E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.135289E-02 | lm-token-buffer-mib: 2.331225E+02 | global_load_balancing_loss: 1.001549E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11232.885 | params rms: 0.060822 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:52:03.616517 | finish at 2026-10-17 18:54:06 + [2026-10-11 17:05:48] iteration 53/ 2384 | consumed samples: 1696 | elapsed time per iteration (ms): 225707.5 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 74331.6 | valid tokens/s: 74331.6 | physical tokens/s: 75409.2 | loss tokens/s: 55944.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.571343E-01 | lm-global-supervised-tokens: 1.262710E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.611079E-02 | lm-token-buffer-mib: 2.334373E+02 | global_load_balancing_loss: 1.000478E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11232.823 | params rms: 0.060822 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:08:44.250813 | finish at 2026-10-17 19:14:32 + [2026-10-11 17:09:30] iteration 54/ 2384 | consumed samples: 1728 | elapsed time per iteration (ms): 221991.9 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 75575.7 | valid tokens/s: 75575.7 | physical tokens/s: 76174.0 | loss tokens/s: 54214.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.938254E-01 | lm-global-supervised-tokens: 1.203527E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.558147E-02 | lm-token-buffer-mib: 2.336797E+02 | global_load_balancing_loss: 1.002304E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11232.759 | params rms: 0.060822 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:40:41.078594 | finish at 2026-10-17 16:50:11 + [2026-10-11 17:13:21] iteration 55/ 2384 | consumed samples: 1760 | elapsed time per iteration (ms): 230990.4 | throughput per GPU (TFLOP/s/GPU): 25.3 | MFU 8.10% | FLOPs estimator: workload-aware | logical tokens/s: 72631.5 | valid tokens/s: 72631.5 | physical tokens/s: 73106.3 | loss tokens/s: 55184.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.645530E-01 | lm-global-supervised-tokens: 1.274705E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.443179E-02 | lm-token-buffer-mib: 2.344899E+02 | global_load_balancing_loss: 1.001004E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11232.696 | params rms: 0.060821 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:26:16.629561 | finish at 2026-10-17 22:39:37 + [2026-10-11 17:17:03] iteration 56/ 2384 | consumed samples: 1792 | elapsed time per iteration (ms): 222843.9 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.58% | FLOPs estimator: workload-aware | logical tokens/s: 75286.7 | valid tokens/s: 75286.7 | physical tokens/s: 75644.7 | loss tokens/s: 54235.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.678661E-01 | lm-global-supervised-tokens: 1.208595E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.197685E-02 | lm-token-buffer-mib: 2.333142E+02 | global_load_balancing_loss: 1.001080E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11232.635 | params rms: 0.060821 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:06:20.559711 | finish at 2026-10-17 17:23:24 + [2026-10-11 17:20:53] iteration 57/ 2384 | consumed samples: 1824 | elapsed time per iteration (ms): 229558.2 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 73084.8 | valid tokens/s: 73084.8 | physical tokens/s: 73816.4 | loss tokens/s: 57685.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.307198E-01 | lm-global-supervised-tokens: 1.324221E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.270923E-02 | lm-token-buffer-mib: 2.359860E+02 | global_load_balancing_loss: 9.989684E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11232.574 | params rms: 0.060821 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:23:02.014901 | finish at 2026-10-17 21:43:55 + [2026-10-11 17:24:35] iteration 58/ 2384 | consumed samples: 1856 | elapsed time per iteration (ms): 221531.8 | throughput per GPU (TFLOP/s/GPU): 17.8 | MFU 5.70% | FLOPs estimator: workload-aware | logical tokens/s: 75732.7 | valid tokens/s: 75732.7 | physical tokens/s: 76908.3 | loss tokens/s: 59826.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.619570E-01 | lm-global-supervised-tokens: 1.325357E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.396109E-02 | lm-token-buffer-mib: 2.366273E+02 | global_load_balancing_loss: 1.000546E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11232.512 | params rms: 0.060820 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:08:03.030094 | finish at 2026-10-17 16:32:38 + [2026-10-11 17:28:16] iteration 59/ 2384 | consumed samples: 1888 | elapsed time per iteration (ms): 221489.4 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75747.1 | valid tokens/s: 75747.1 | physical tokens/s: 76724.5 | loss tokens/s: 61966.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.322217E-01 | lm-global-supervised-tokens: 1.372491E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.108568E-02 | lm-token-buffer-mib: 2.351994E+02 | global_load_balancing_loss: 1.000410E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11232.450 | params rms: 0.060820 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:02:42.966210 | finish at 2026-10-17 16:30:59 + [2026-10-11 17:32:16] iteration 60/ 2384 | consumed samples: 1920 | elapsed time per iteration (ms): 239858.1 | throughput per GPU (TFLOP/s/GPU): 17.3 | MFU 5.56% | FLOPs estimator: workload-aware | logical tokens/s: 69946.4 | valid tokens/s: 69946.4 | physical tokens/s: 70776.5 | loss tokens/s: 51515.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.044330E-01 | lm-global-supervised-tokens: 1.235637E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.502503E-02 | lm-token-buffer-mib: 2.350911E+02 | global_load_balancing_loss: 9.997380E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11232.389 | params rms: 0.060820 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:50:30.142804 | finish at 2026-10-18 04:22:46 + [2026-10-11 17:36:00] iteration 61/ 2384 | consumed samples: 1952 | elapsed time per iteration (ms): 223901.3 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 74931.1 | valid tokens/s: 74931.1 | physical tokens/s: 76024.7 | loss tokens/s: 52967.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.230019E-01 | lm-global-supervised-tokens: 1.185954E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.749394E-02 | lm-token-buffer-mib: 2.364337E+02 | global_load_balancing_loss: 1.004082E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11232.332 | params rms: 0.060819 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:28:42.610130 | finish at 2026-10-17 18:04:42 + [2026-10-11 17:39:41] iteration 62/ 2384 | consumed samples: 1984 | elapsed time per iteration (ms): 221425.6 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75769.0 | valid tokens/s: 75769.0 | physical tokens/s: 76721.7 | loss tokens/s: 58042.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.106827E-01 | lm-global-supervised-tokens: 1.285203E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.850157E-02 | lm-token-buffer-mib: 2.343329E+02 | global_load_balancing_loss: 1.001474E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11232.274 | params rms: 0.060819 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:49:10.325254 | finish at 2026-10-17 16:28:52 + [2026-10-11 17:43:23] iteration 63/ 2384 | consumed samples: 2016 | elapsed time per iteration (ms): 221359.2 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.45% | FLOPs estimator: workload-aware | logical tokens/s: 75791.8 | valid tokens/s: 75791.8 | physical tokens/s: 76689.5 | loss tokens/s: 58723.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.893934E-01 | lm-global-supervised-tokens: 1.299897E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.444045E-02 | lm-token-buffer-mib: 2.360183E+02 | global_load_balancing_loss: 9.993465E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11232.216 | params rms: 0.060819 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:42:54.697115 | finish at 2026-10-17 16:26:17 + [2026-10-11 17:47:06] iteration 64/ 2384 | consumed samples: 2048 | elapsed time per iteration (ms): 223290.5 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75136.2 | valid tokens/s: 75136.2 | physical tokens/s: 75794.6 | loss tokens/s: 59293.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.610875E-01 | lm-global-supervised-tokens: 1.323959E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.672171E-02 | lm-token-buffer-mib: 2.353088E+02 | global_load_balancing_loss: 9.996882E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11232.159 | params rms: 0.060818 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:53:53.955956 | finish at 2026-10-17 17:41:00 + [2026-10-11 17:50:49] iteration 65/ 2384 | consumed samples: 2080 | elapsed time per iteration (ms): 223132.5 | throughput per GPU (TFLOP/s/GPU): 27.2 | MFU 8.71% | FLOPs estimator: workload-aware | logical tokens/s: 75189.4 | valid tokens/s: 75189.4 | physical tokens/s: 75427.2 | loss tokens/s: 56771.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.839136E-01 | lm-global-supervised-tokens: 1.266760E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.523274E-02 | lm-token-buffer-mib: 2.328889E+02 | global_load_balancing_loss: 9.990514E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11232.099 | params rms: 0.060818 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:44:04.330375 | finish at 2026-10-17 17:34:53 + [2026-10-11 17:54:33] iteration 66/ 2384 | consumed samples: 2112 | elapsed time per iteration (ms): 223993.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 74900.3 | valid tokens/s: 74900.3 | physical tokens/s: 75445.6 | loss tokens/s: 56056.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.422974E-01 | lm-global-supervised-tokens: 1.255621E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.349815E-02 | lm-token-buffer-mib: 2.345027E+02 | global_load_balancing_loss: 1.000912E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11232.033 | params rms: 0.060818 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:13:36.831851 | finish at 2026-10-17 18:08:10 + [2026-10-11 17:58:17] iteration 67/ 2384 | consumed samples: 2144 | elapsed time per iteration (ms): 224326.6 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 74789.1 | valid tokens/s: 74789.0 | physical tokens/s: 75382.1 | loss tokens/s: 56142.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.238558E-01 | lm-global-supervised-tokens: 1.259421E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.437055E-02 | lm-token-buffer-mib: 2.349483E+02 | global_load_balancing_loss: 1.002108E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11231.966 | params rms: 0.060817 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:22:44.663321 | finish at 2026-10-17 18:21:02 + [2026-10-11 18:02:04] iteration 68/ 2384 | consumed samples: 2176 | elapsed time per iteration (ms): 226374.2 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 74112.7 | valid tokens/s: 74112.7 | physical tokens/s: 74877.8 | loss tokens/s: 56575.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.539176E-01 | lm-global-supervised-tokens: 1.280730E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.055442E-02 | lm-token-buffer-mib: 2.333818E+02 | global_load_balancing_loss: 1.000516E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11231.902 | params rms: 0.060817 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:38:02.538666 | finish at 2026-10-17 19:40:06 + [2026-10-11 18:05:47] iteration 69/ 2384 | consumed samples: 2208 | elapsed time per iteration (ms): 222931.7 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 75257.1 | valid tokens/s: 75257.1 | physical tokens/s: 76321.1 | loss tokens/s: 52948.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.417812E-01 | lm-global-supervised-tokens: 1.180386E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.360563E-02 | lm-token-buffer-mib: 2.347101E+02 | global_load_balancing_loss: 1.000982E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11231.841 | params rms: 0.060817 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:21:26.963303 | finish at 2026-10-17 17:27:14 + [2026-10-11 18:09:44] iteration 70/ 2384 | consumed samples: 2240 | elapsed time per iteration (ms): 237136.2 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 70749.2 | valid tokens/s: 70749.2 | physical tokens/s: 71425.6 | loss tokens/s: 51513.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.772311E-01 | lm-global-supervised-tokens: 1.221563E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.487821E-02 | lm-token-buffer-mib: 2.327875E+02 | global_load_balancing_loss: 1.001385E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11231.780 | params rms: 0.060816 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:25:33.107655 | finish at 2026-10-18 02:35:17 + [2026-10-11 18:13:27] iteration 71/ 2384 | consumed samples: 2272 | elapsed time per iteration (ms): 222785.4 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.06% | FLOPs estimator: workload-aware | logical tokens/s: 75306.6 | valid tokens/s: 75306.6 | physical tokens/s: 76353.7 | loss tokens/s: 59690.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.186094E-01 | lm-global-supervised-tokens: 1.329816E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.369280E-02 | lm-token-buffer-mib: 2.354453E+02 | global_load_balancing_loss: 1.000712E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11231.720 | params rms: 0.060816 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:08:22.529634 | finish at 2026-10-17 17:21:49 + [2026-10-11 18:17:08] iteration 72/ 2384 | consumed samples: 2304 | elapsed time per iteration (ms): 221139.2 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 75867.2 | valid tokens/s: 75867.2 | physical tokens/s: 76766.4 | loss tokens/s: 56472.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.105680E-01 | lm-global-supervised-tokens: 1.248833E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.455913E-02 | lm-token-buffer-mib: 2.342398E+02 | global_load_balancing_loss: 1.000005E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11231.658 | params rms: 0.060816 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:01:13.751511 | finish at 2026-10-17 16:18:21 + [2026-10-11 18:20:47] iteration 73/ 2384 | consumed samples: 2336 | elapsed time per iteration (ms): 219764.8 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.26% | FLOPs estimator: workload-aware | logical tokens/s: 76341.6 | valid tokens/s: 76341.6 | physical tokens/s: 76824.8 | loss tokens/s: 59970.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.629897E-01 | lm-global-supervised-tokens: 1.317930E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.378840E-02 | lm-token-buffer-mib: 2.336733E+02 | global_load_balancing_loss: 1.000804E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11231.595 | params rms: 0.060815 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:04:36.417152 | finish at 2026-10-17 15:25:24 + [2026-10-11 18:24:29] iteration 74/ 2384 | consumed samples: 2368 | elapsed time per iteration (ms): 221180.8 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 75852.9 | valid tokens/s: 75852.9 | physical tokens/s: 76391.1 | loss tokens/s: 56387.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.728871E-01 | lm-global-supervised-tokens: 1.247187E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.935239E-02 | lm-token-buffer-mib: 2.333587E+02 | global_load_balancing_loss: 1.001362E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11231.535 | params rms: 0.060815 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:55:27.688115 | finish at 2026-10-17 16:19:56 + [2026-10-11 18:28:09] iteration 75/ 2384 | consumed samples: 2400 | elapsed time per iteration (ms): 220650.3 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 76035.2 | valid tokens/s: 76035.2 | physical tokens/s: 76681.0 | loss tokens/s: 58672.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.899670E-01 | lm-global-supervised-tokens: 1.294602E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.005864E-02 | lm-token-buffer-mib: 2.333930E+02 | global_load_balancing_loss: 1.000476E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11231.475 | params rms: 0.060815 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:31:21.594508 | finish at 2026-10-17 15:59:31 + [2026-10-11 18:31:49] iteration 76/ 2384 | consumed samples: 2432 | elapsed time per iteration (ms): 219540.4 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 76419.6 | valid tokens/s: 76419.6 | physical tokens/s: 77199.5 | loss tokens/s: 56004.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.277272E-01 | lm-global-supervised-tokens: 1.229525E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.664651E-02 | lm-token-buffer-mib: 2.339110E+02 | global_load_balancing_loss: 1.001127E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11231.414 | params rms: 0.060814 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:44:59.276281 | finish at 2026-10-17 15:16:48 + [2026-10-11 18:35:30] iteration 77/ 2384 | consumed samples: 2464 | elapsed time per iteration (ms): 221302.9 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75811.0 | valid tokens/s: 75811.0 | physical tokens/s: 76885.7 | loss tokens/s: 58454.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.361924E-01 | lm-global-supervised-tokens: 1.293623E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.015193E-02 | lm-token-buffer-mib: 2.352722E+02 | global_load_balancing_loss: 1.001489E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11231.353 | params rms: 0.060814 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:49:05.872980 | finish at 2026-10-17 16:24:36 + [2026-10-11 18:39:17] iteration 78/ 2384 | consumed samples: 2496 | elapsed time per iteration (ms): 227194.6 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 73845.0 | valid tokens/s: 73845.0 | physical tokens/s: 74590.2 | loss tokens/s: 54919.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.122805E-01 | lm-global-supervised-tokens: 1.247736E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.193914E-02 | lm-token-buffer-mib: 2.346710E+02 | global_load_balancing_loss: 9.997762E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11231.293 | params rms: 0.060814 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:31:50.678569 | finish at 2026-10-17 20:11:08 + [2026-10-11 18:43:00] iteration 79/ 2384 | consumed samples: 2528 | elapsed time per iteration (ms): 222234.8 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 75493.1 | valid tokens/s: 75493.1 | physical tokens/s: 75976.4 | loss tokens/s: 59353.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.396019E-01 | lm-global-supervised-tokens: 1.319048E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.269524E-02 | lm-token-buffer-mib: 2.325827E+02 | global_load_balancing_loss: 9.994640E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11231.232 | params rms: 0.060813 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:17:31.177961 | finish at 2026-10-17 17:00:31 + [2026-10-11 18:46:58] iteration 80/ 2384 | consumed samples: 2560 | elapsed time per iteration (ms): 238242.4 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 70420.7 | valid tokens/s: 70420.7 | physical tokens/s: 71262.9 | loss tokens/s: 56563.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.715167E-01 | lm-global-supervised-tokens: 1.347582E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.161023E-02 | lm-token-buffer-mib: 2.341868E+02 | global_load_balancing_loss: 9.989673E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11231.170 | params rms: 0.060813 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:28:30.488892 | finish at 2026-10-18 03:15:28 +W1011 18:47:34.988000 541120 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 18:47:34.988000 541120 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +W1011 18:47:35.009000 541125 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1011 18:47:35.009000 541125 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +times across ranks (ms): + save-checkpoint: + rank 0: 105960.05 + rank 1: 105960.07 + rank 2: 105960.06 + rank 3: 105959.99 + rank 4: 105959.97 + rank 5: 105960.07 + rank 6: 105960.01 + rank 7: 105959.98 + rank 8: 105962.69 + rank 9: 105962.61 + rank 10: 105962.61 + rank 11: 105962.68 + rank 12: 105962.70 + rank 13: 105962.69 + rank 14: 105962.70 + rank 15: 105962.70 + rank 16: 105960.41 + rank 17: 105960.31 + rank 18: 105960.31 + rank 19: 105960.32 + rank 20: 105960.33 + rank 21: 105960.38 + rank 22: 105960.34 + rank 23: 105960.32 + rank 24: 105959.09 + rank 25: 105959.09 + rank 26: 105959.08 + rank 27: 105959.09 + rank 28: 105959.06 + rank 29: 105959.12 + rank 30: 105959.09 + rank 31: 105959.08 + rank 32: 105957.57 + rank 33: 105957.56 + rank 34: 105957.57 + rank 35: 105957.57 + rank 36: 105957.57 + rank 37: 105957.57 + rank 38: 105957.68 + rank 39: 105957.60 + rank 40: 105958.96 + rank 41: 105959.12 + rank 42: 105958.98 + rank 43: 105958.97 + rank 44: 105959.02 + rank 45: 105959.04 + rank 46: 105959.02 + rank 47: 105958.95 + rank 48: 105959.84 + rank 49: 105959.85 + rank 50: 105959.87 + rank 51: 105959.84 + rank 52: 105959.84 + rank 53: 105959.83 + rank 54: 105959.84 + rank 55: 105959.84 + rank 56: 105958.18 + rank 57: 105958.20 + rank 58: 105958.20 + rank 59: 105958.20 + rank 60: 105958.20 + rank 61: 105958.18 + rank 62: 105958.17 + rank 63: 105958.17 + [2026-10-11 18:52:21] iteration 81/ 2384 | consumed samples: 2592 | elapsed time per iteration (ms): 217125.4 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 77269.5 | valid tokens/s: 77269.5 | physical tokens/s: 78076.8 | loss tokens/s: 61044.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.019277E-01 | lm-global-supervised-tokens: 1.325423E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.973375E-02 | lm-token-buffer-mib: 2.339876E+02 | global_load_balancing_loss: 1.003084E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11231.105 | params rms: 0.060813 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:53:59.682145 | finish at 2026-10-17 13:46:21 + [2026-10-11 18:56:10] iteration 82/ 2384 | consumed samples: 2624 | elapsed time per iteration (ms): 229065.9 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 73241.7 | valid tokens/s: 73241.7 | physical tokens/s: 74169.2 | loss tokens/s: 56003.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.818955E-01 | lm-global-supervised-tokens: 1.282844E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.483049E-02 | lm-token-buffer-mib: 2.353244E+02 | global_load_balancing_loss: 9.997398E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11231.039 | params rms: 0.060812 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:28:29.795853 | finish at 2026-10-17 21:24:40 + [2026-10-11 18:59:55] iteration 83/ 2384 | consumed samples: 2656 | elapsed time per iteration (ms): 224734.9 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 74653.3 | valid tokens/s: 74653.3 | physical tokens/s: 75717.3 | loss tokens/s: 58796.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.765146E-01 | lm-global-supervised-tokens: 1.321370E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.351702E-02 | lm-token-buffer-mib: 2.361575E+02 | global_load_balancing_loss: 9.997645E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11230.972 | params rms: 0.060812 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:38:34.924799 | finish at 2026-10-17 18:38:30 + [2026-10-11 19:03:40] iteration 84/ 2384 | consumed samples: 2688 | elapsed time per iteration (ms): 224964.9 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 74576.9 | valid tokens/s: 74576.9 | physical tokens/s: 75279.7 | loss tokens/s: 56671.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.815928E-01 | lm-global-supervised-tokens: 1.274904E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.879406E-02 | lm-token-buffer-mib: 2.334554E+02 | global_load_balancing_loss: 1.000990E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11230.906 | params rms: 0.060812 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:43:39.382453 | finish at 2026-10-17 18:47:19 + [2026-10-11 19:07:21] iteration 85/ 2384 | consumed samples: 2720 | elapsed time per iteration (ms): 221305.4 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75810.0 | valid tokens/s: 75810.0 | physical tokens/s: 76838.3 | loss tokens/s: 55896.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.539368E-01 | lm-global-supervised-tokens: 1.237013E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.373177E-02 | lm-token-buffer-mib: 2.343750E+02 | global_load_balancing_loss: 1.001164E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11230.838 | params rms: 0.060811 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:19:41.213568 | finish at 2026-10-17 16:27:02 + [2026-10-11 19:11:04] iteration 86/ 2384 | consumed samples: 2752 | elapsed time per iteration (ms): 223043.8 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 75219.3 | valid tokens/s: 75219.3 | physical tokens/s: 76730.7 | loss tokens/s: 58014.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.451549E-01 | lm-global-supervised-tokens: 1.293969E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.238102E-02 | lm-token-buffer-mib: 2.388286E+02 | global_load_balancing_loss: 1.000505E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11230.768 | params rms: 0.060811 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:22:34.755668 | finish at 2026-10-17 17:33:39 + [2026-10-11 19:14:43] iteration 87/ 2384 | consumed samples: 2784 | elapsed time per iteration (ms): 218927.6 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 76633.4 | valid tokens/s: 76633.4 | physical tokens/s: 77087.6 | loss tokens/s: 60976.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.852182E-01 | lm-global-supervised-tokens: 1.334953E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.316172E-02 | lm-token-buffer-mib: 2.335181E+02 | global_load_balancing_loss: 1.000250E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11230.702 | params rms: 0.060811 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:41:16.663032 | finish at 2026-10-17 14:56:00 + [2026-10-11 19:18:24] iteration 88/ 2384 | consumed samples: 2816 | elapsed time per iteration (ms): 221290.1 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75815.5 | valid tokens/s: 75815.5 | physical tokens/s: 76730.3 | loss tokens/s: 55410.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.881902E-01 | lm-global-supervised-tokens: 1.226176E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.365034E-02 | lm-token-buffer-mib: 2.349746E+02 | global_load_balancing_loss: 1.001308E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11230.636 | params rms: 0.060810 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:08:02.047380 | finish at 2026-10-17 16:26:26 + [2026-10-11 19:22:11] iteration 89/ 2384 | consumed samples: 2848 | elapsed time per iteration (ms): 226612.5 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 74034.8 | valid tokens/s: 74034.8 | physical tokens/s: 74771.6 | loss tokens/s: 58947.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.437210E-01 | lm-global-supervised-tokens: 1.335834E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.465213E-02 | lm-token-buffer-mib: 2.346078E+02 | global_load_balancing_loss: 1.000398E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11230.571 | params rms: 0.060810 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:27:55.782598 | finish at 2026-10-17 19:50:07 + [2026-10-11 19:26:14] iteration 90/ 2384 | consumed samples: 2880 | elapsed time per iteration (ms): 243236.2 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 68974.9 | valid tokens/s: 68974.9 | physical tokens/s: 69602.5 | loss tokens/s: 50679.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.733403E-01 | lm-global-supervised-tokens: 1.232710E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.262643E-02 | lm-token-buffer-mib: 2.342825E+02 | global_load_balancing_loss: 1.001585E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11230.505 | params rms: 0.060810 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:59:43.880754 | finish at 2026-10-18 06:25:58 + [2026-10-11 19:30:02] iteration 91/ 2384 | consumed samples: 2912 | elapsed time per iteration (ms): 228375.8 | throughput per GPU (TFLOP/s/GPU): 17.6 | MFU 5.66% | FLOPs estimator: workload-aware | logical tokens/s: 73463.2 | valid tokens/s: 73463.2 | physical tokens/s: 74523.3 | loss tokens/s: 59455.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.236550E-01 | lm-global-supervised-tokens: 1.357830E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.747432E-02 | lm-token-buffer-mib: 2.359829E+02 | global_load_balancing_loss: 1.000728E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11230.438 | params rms: 0.060809 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:27:45.671436 | finish at 2026-10-17 20:57:48 + [2026-10-11 19:33:43] iteration 92/ 2384 | consumed samples: 2944 | elapsed time per iteration (ms): 220997.0 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75916.0 | valid tokens/s: 75916.0 | physical tokens/s: 77069.5 | loss tokens/s: 58515.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.476588E-01 | lm-global-supervised-tokens: 1.293172E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.096402E-02 | lm-token-buffer-mib: 2.344597E+02 | global_load_balancing_loss: 1.000704E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11230.373 | params rms: 0.060809 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:42:05.187341 | finish at 2026-10-17 16:15:49 + [2026-10-11 19:37:29] iteration 93/ 2384 | consumed samples: 2976 | elapsed time per iteration (ms): 225670.0 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.76% | FLOPs estimator: workload-aware | logical tokens/s: 74343.7 | valid tokens/s: 74343.7 | physical tokens/s: 74921.4 | loss tokens/s: 54920.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.514231E-01 | lm-global-supervised-tokens: 1.239392E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.932313E-02 | lm-token-buffer-mib: 2.337751E+02 | global_load_balancing_loss: 1.002926E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11230.307 | params rms: 0.060808 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:36:50.021519 | finish at 2026-10-17 19:14:19 + [2026-10-11 19:41:09] iteration 94/ 2384 | consumed samples: 3008 | elapsed time per iteration (ms): 219764.9 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.66% | FLOPs estimator: workload-aware | logical tokens/s: 76341.6 | valid tokens/s: 76341.6 | physical tokens/s: 76669.6 | loss tokens/s: 58850.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.429054E-01 | lm-global-supervised-tokens: 1.293327E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.935369E-02 | lm-token-buffer-mib: 2.332067E+02 | global_load_balancing_loss: 1.002276E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11230.240 | params rms: 0.060808 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:47:41.632395 | finish at 2026-10-17 15:28:51 + [2026-10-11 19:44:51] iteration 95/ 2384 | consumed samples: 3040 | elapsed time per iteration (ms): 222173.1 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.17% | FLOPs estimator: workload-aware | logical tokens/s: 75514.1 | valid tokens/s: 75514.1 | physical tokens/s: 76364.2 | loss tokens/s: 57743.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.383941E-01 | lm-global-supervised-tokens: 1.282900E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.605413E-02 | lm-token-buffer-mib: 2.363120E+02 | global_load_balancing_loss: 9.994134E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11230.174 | params rms: 0.060808 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:15:54.252629 | finish at 2026-10-17 17:00:45 + [2026-10-11 19:48:32] iteration 96/ 2384 | consumed samples: 3072 | elapsed time per iteration (ms): 221027.0 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75905.6 | valid tokens/s: 75905.6 | physical tokens/s: 76265.1 | loss tokens/s: 59787.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.095393E-01 | lm-global-supervised-tokens: 1.321454E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.206619E-02 | lm-token-buffer-mib: 2.329427E+02 | global_load_balancing_loss: 1.000370E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11230.108 | params rms: 0.060807 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:28:29.850075 | finish at 2026-10-17 16:17:02 + [2026-10-11 19:52:21] iteration 97/ 2384 | consumed samples: 3104 | elapsed time per iteration (ms): 228520.8 | throughput per GPU (TFLOP/s/GPU): 26.8 | MFU 8.58% | FLOPs estimator: workload-aware | logical tokens/s: 73416.4 | valid tokens/s: 73416.4 | physical tokens/s: 74337.1 | loss tokens/s: 53782.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.766208E-01 | lm-global-supervised-tokens: 1.229049E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.490897E-02 | lm-token-buffer-mib: 2.359875E+02 | global_load_balancing_loss: 1.000557E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11230.041 | params rms: 0.060807 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:10:27.180548 | finish at 2026-10-17 21:02:48 + [2026-10-11 19:56:04] iteration 98/ 2384 | consumed samples: 3136 | elapsed time per iteration (ms): 223800.8 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 74964.9 | valid tokens/s: 74964.9 | physical tokens/s: 75358.2 | loss tokens/s: 56997.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.912655E-01 | lm-global-supervised-tokens: 1.275606E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.788471E-02 | lm-token-buffer-mib: 2.329099E+02 | global_load_balancing_loss: 1.000011E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11229.975 | params rms: 0.060807 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:06:48.589208 | finish at 2026-10-17 18:02:53 + [2026-10-11 19:59:46] iteration 99/ 2384 | consumed samples: 3168 | elapsed time per iteration (ms): 221660.6 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 75688.6 | valid tokens/s: 75688.6 | physical tokens/s: 76841.7 | loss tokens/s: 61206.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.695986E-01 | lm-global-supervised-tokens: 1.356704E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.687880E-02 | lm-token-buffer-mib: 2.353491E+02 | global_load_balancing_loss: 1.000999E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11229.912 | params rms: 0.060806 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:41:34.519365 | finish at 2026-10-17 16:41:21 + [2026-10-11 20:03:44] iteration 100/ 2384 | consumed samples: 3200 | elapsed time per iteration (ms): 237919.7 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 70516.2 | valid tokens/s: 70516.2 | physical tokens/s: 71064.3 | loss tokens/s: 53180.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.981089E-01 | lm-global-supervised-tokens: 1.265271E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.611489E-02 | lm-token-buffer-mib: 2.346313E+02 | global_load_balancing_loss: 1.000186E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11229.848 | params rms: 0.060806 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:56:48.528153 | finish at 2026-10-18 03:00:33 + [2026-10-11 20:07:25] iteration 101/ 2384 | consumed samples: 3232 | elapsed time per iteration (ms): 221250.2 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 75829.1 | valid tokens/s: 75829.1 | physical tokens/s: 76503.1 | loss tokens/s: 56938.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.319414E-01 | lm-global-supervised-tokens: 1.259754E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.485626E-02 | lm-token-buffer-mib: 2.347661E+02 | global_load_balancing_loss: 9.999180E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11229.786 | params rms: 0.060806 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:18:34.220828 | finish at 2026-10-17 16:25:59 + [2026-10-11 20:11:06] iteration 102/ 2384 | consumed samples: 3264 | elapsed time per iteration (ms): 220799.9 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 75983.6 | valid tokens/s: 75983.6 | physical tokens/s: 76320.9 | loss tokens/s: 54802.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.650785E-01 | lm-global-supervised-tokens: 1.210027E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.913318E-02 | lm-token-buffer-mib: 2.333345E+02 | global_load_balancing_loss: 1.000279E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11229.721 | params rms: 0.060805 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:57:45.339281 | finish at 2026-10-17 16:08:51 + [2026-10-11 20:14:56] iteration 103/ 2384 | consumed samples: 3296 | elapsed time per iteration (ms): 229619.8 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.03% | FLOPs estimator: workload-aware | logical tokens/s: 73065.2 | valid tokens/s: 73065.2 | physical tokens/s: 74324.7 | loss tokens/s: 59450.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.284775E-01 | lm-global-supervised-tokens: 1.365106E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.128989E-02 | lm-token-buffer-mib: 2.349807E+02 | global_load_balancing_loss: 1.001834E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11229.654 | params rms: 0.060805 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:29:22.729202 | finish at 2026-10-17 21:44:18 + [2026-10-11 20:18:37] iteration 104/ 2384 | consumed samples: 3328 | elapsed time per iteration (ms): 221789.8 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 75644.6 | valid tokens/s: 75644.6 | physical tokens/s: 76363.8 | loss tokens/s: 59328.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.687084E-01 | lm-global-supervised-tokens: 1.315848E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.416581E-02 | lm-token-buffer-mib: 2.342785E+02 | global_load_balancing_loss: 1.002093E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11229.591 | params rms: 0.060805 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:28:00.742750 | finish at 2026-10-17 16:46:38 + [2026-10-11 20:22:30] iteration 105/ 2384 | consumed samples: 3360 | elapsed time per iteration (ms): 232548.8 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.25% | FLOPs estimator: workload-aware | logical tokens/s: 72144.7 | valid tokens/s: 72144.7 | physical tokens/s: 72809.8 | loss tokens/s: 55150.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 1.295711E+00 | lm-global-supervised-tokens: 1.282510E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.627463E-02 | lm-token-buffer-mib: 2.346182E+02 | global_load_balancing_loss: 1.002616E+00 | loss scale: 1.0 | grad norm: 19.985 | params norm: 11229.525 | params rms: 0.060804 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:12:58.766800 | finish at 2026-10-17 23:35:29 + [2026-10-11 20:26:13] iteration 106/ 2384 | consumed samples: 3392 | elapsed time per iteration (ms): 223374.8 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 75107.8 | valid tokens/s: 75107.8 | physical tokens/s: 75582.1 | loss tokens/s: 53197.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.872565E-01 | lm-global-supervised-tokens: 1.188306E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.749618E-02 | lm-token-buffer-mib: 2.332198E+02 | global_load_balancing_loss: 1.003287E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11229.441 | params rms: 0.060804 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:20:47.813877 | finish at 2026-10-17 17:47:01 + [2026-10-11 20:29:54] iteration 107/ 2384 | consumed samples: 3424 | elapsed time per iteration (ms): 220966.5 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 75926.4 | valid tokens/s: 75926.4 | physical tokens/s: 76515.3 | loss tokens/s: 58284.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.818594E-01 | lm-global-supervised-tokens: 1.287887E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.921105E-02 | lm-token-buffer-mib: 2.330961E+02 | global_load_balancing_loss: 1.003426E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11229.379 | params rms: 0.060803 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:45:40.833519 | finish at 2026-10-17 16:15:35 + [2026-10-11 20:33:37] iteration 108/ 2384 | consumed samples: 3456 | elapsed time per iteration (ms): 222420.4 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 75430.0 | valid tokens/s: 75430.0 | physical tokens/s: 76177.5 | loss tokens/s: 57148.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.803079E-01 | lm-global-supervised-tokens: 1.271095E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.353657E-02 | lm-token-buffer-mib: 2.339656E+02 | global_load_balancing_loss: 1.001889E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11229.331 | params rms: 0.060803 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:37:08.780802 | finish at 2026-10-17 17:10:46 + [2026-10-11 20:37:21] iteration 109/ 2384 | consumed samples: 3488 | elapsed time per iteration (ms): 224053.2 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.21% | FLOPs estimator: workload-aware | logical tokens/s: 74880.4 | valid tokens/s: 74880.4 | physical tokens/s: 76407.3 | loss tokens/s: 54090.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.201604E-01 | lm-global-supervised-tokens: 1.211910E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.716265E-02 | lm-token-buffer-mib: 2.365849E+02 | global_load_balancing_loss: 1.002721E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11229.291 | params rms: 0.060803 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:35:21.139580 | finish at 2026-10-17 18:12:42 + [2026-10-11 20:41:22] iteration 110/ 2384 | consumed samples: 3520 | elapsed time per iteration (ms): 241535.0 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 69460.6 | valid tokens/s: 69460.6 | physical tokens/s: 70318.8 | loss tokens/s: 53753.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.048456E-01 | lm-global-supervised-tokens: 1.298324E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.792133E-02 | lm-token-buffer-mib: 2.363940E+02 | global_load_balancing_loss: 9.992131E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11229.258 | params rms: 0.060803 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:34:10.568509 | finish at 2026-10-18 05:15:33 + [2026-10-11 20:45:10] iteration 111/ 2384 | consumed samples: 3552 | elapsed time per iteration (ms): 228025.2 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.80% | FLOPs estimator: workload-aware | logical tokens/s: 73576.0 | valid tokens/s: 73576.0 | physical tokens/s: 74056.1 | loss tokens/s: 59972.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.915825E-01 | lm-global-supervised-tokens: 1.367516E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.388277E-02 | lm-token-buffer-mib: 2.335115E+02 | global_load_balancing_loss: 1.001333E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11229.229 | params rms: 0.060803 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:58:21.224074 | finish at 2026-10-17 20:43:32 + [2026-10-11 20:48:51] iteration 112/ 2384 | consumed samples: 3584 | elapsed time per iteration (ms): 220586.7 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 76057.0 | valid tokens/s: 76057.0 | physical tokens/s: 76761.7 | loss tokens/s: 60616.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.360054E-01 | lm-global-supervised-tokens: 1.337126E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.232684E-02 | lm-token-buffer-mib: 2.332435E+02 | global_load_balancing_loss: 9.984341E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11229.205 | params rms: 0.060802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:12:53.081985 | finish at 2026-10-17 16:01:44 + [2026-10-11 20:52:32] iteration 113/ 2384 | consumed samples: 3616 | elapsed time per iteration (ms): 220828.6 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 75973.9 | valid tokens/s: 75973.8 | physical tokens/s: 76328.1 | loss tokens/s: 55826.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.979640E-01 | lm-global-supervised-tokens: 1.232804E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.050495E-02 | lm-token-buffer-mib: 2.335573E+02 | global_load_balancing_loss: 1.001771E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11229.181 | params rms: 0.060802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:18:21.642692 | finish at 2026-10-17 16:10:53 + [2026-10-11 20:56:17] iteration 114/ 2384 | consumed samples: 3648 | elapsed time per iteration (ms): 225513.7 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 74395.5 | valid tokens/s: 74395.5 | physical tokens/s: 75068.2 | loss tokens/s: 52875.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.417013E-01 | lm-global-supervised-tokens: 1.192417E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.153073E-02 | lm-token-buffer-mib: 2.344773E+02 | global_load_balancing_loss: 9.993522E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11229.157 | params rms: 0.060802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:11:56.042733 | finish at 2026-10-17 19:08:13 + [2026-10-11 21:00:01] iteration 115/ 2384 | consumed samples: 3680 | elapsed time per iteration (ms): 223836.1 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 74953.1 | valid tokens/s: 74953.1 | physical tokens/s: 75492.7 | loss tokens/s: 60480.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.639861E-01 | lm-global-supervised-tokens: 1.353760E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.598996E-02 | lm-token-buffer-mib: 2.340760E+02 | global_load_balancing_loss: 1.000683E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11229.131 | params rms: 0.060802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:04:44.043509 | finish at 2026-10-17 18:04:45 + [2026-10-11 21:03:44] iteration 116/ 2384 | consumed samples: 3712 | elapsed time per iteration (ms): 222806.5 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.88% | FLOPs estimator: workload-aware | logical tokens/s: 75299.3 | valid tokens/s: 75299.3 | physical tokens/s: 75774.6 | loss tokens/s: 58488.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.893243E-01 | lm-global-supervised-tokens: 1.303157E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.841263E-02 | lm-token-buffer-mib: 2.341477E+02 | global_load_balancing_loss: 1.000734E+00 | loss scale: 1.0 | grad norm: 0.338 | params norm: 11229.102 | params rms: 0.060802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:22:05.056469 | finish at 2026-10-17 17:25:49 + [2026-10-11 21:07:26] iteration 117/ 2384 | consumed samples: 3744 | elapsed time per iteration (ms): 221905.5 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 75605.2 | valid tokens/s: 75605.2 | physical tokens/s: 76159.3 | loss tokens/s: 56764.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.106249E-01 | lm-global-supervised-tokens: 1.259637E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.466236E-02 | lm-token-buffer-mib: 2.343653E+02 | global_load_balancing_loss: 9.963008E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11229.058 | params rms: 0.060802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:44:19.748895 | finish at 2026-10-17 16:51:46 + [2026-10-11 21:11:08] iteration 118/ 2384 | consumed samples: 3776 | elapsed time per iteration (ms): 222294.5 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 75472.8 | valid tokens/s: 75472.8 | physical tokens/s: 75942.8 | loss tokens/s: 60214.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.254784E-01 | lm-global-supervised-tokens: 1.338530E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.141220E-02 | lm-token-buffer-mib: 2.333490E+02 | global_load_balancing_loss: 1.001811E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11229.022 | params rms: 0.060802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:55:19.248113 | finish at 2026-10-17 17:06:27 + [2026-10-11 21:14:55] iteration 119/ 2384 | consumed samples: 3808 | elapsed time per iteration (ms): 226752.3 | throughput per GPU (TFLOP/s/GPU): 17.2 | MFU 5.52% | FLOPs estimator: workload-aware | logical tokens/s: 73989.1 | valid tokens/s: 73989.1 | physical tokens/s: 75630.5 | loss tokens/s: 56264.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.724425E-01 | lm-global-supervised-tokens: 1.275819E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.805581E-02 | lm-token-buffer-mib: 2.348567E+02 | global_load_balancing_loss: 1.000255E+00 | loss scale: 1.0 | grad norm: 0.280 | params norm: 11228.992 | params rms: 0.060801 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:39:53.850471 | finish at 2026-10-17 19:54:49 + [2026-10-11 21:18:57] iteration 120/ 2384 | consumed samples: 3840 | elapsed time per iteration (ms): 242092.1 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.48% | FLOPs estimator: workload-aware | logical tokens/s: 69300.9 | valid tokens/s: 69300.9 | physical tokens/s: 70401.0 | loss tokens/s: 49727.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.421905E-01 | lm-global-supervised-tokens: 1.203851E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.272875E-02 | lm-token-buffer-mib: 2.383997E+02 | global_load_balancing_loss: 1.001579E+00 | loss scale: 1.0 | grad norm: 0.193 | params norm: 11228.965 | params rms: 0.060801 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:14:56.544952 | finish at 2026-10-18 05:33:54 +times across ranks (ms): + save-checkpoint: + rank 0: 23255.35 + rank 1: 23255.36 + rank 2: 23255.34 + rank 3: 23255.37 + rank 4: 23255.35 + rank 5: 23255.35 + rank 6: 23255.35 + rank 7: 23255.34 + rank 8: 23255.38 + rank 9: 23255.39 + rank 10: 23255.39 + rank 11: 23255.37 + rank 12: 23255.37 + rank 13: 23255.37 + rank 14: 23255.38 + rank 15: 23255.37 + rank 16: 23255.30 + rank 17: 23255.31 + rank 18: 23255.32 + rank 19: 23255.32 + rank 20: 23255.30 + rank 21: 23255.31 + rank 22: 23255.31 + rank 23: 23255.32 + rank 24: 23255.37 + rank 25: 23255.12 + rank 26: 23255.42 + rank 27: 23255.38 + rank 28: 23255.40 + rank 29: 23255.40 + rank 30: 23255.39 + rank 31: 23255.38 + rank 32: 23255.33 + rank 33: 23255.32 + rank 34: 23255.31 + rank 35: 23255.35 + rank 36: 23255.29 + rank 37: 23255.31 + rank 38: 23255.29 + rank 39: 23255.30 + rank 40: 23255.77 + rank 41: 23255.75 + rank 42: 23255.76 + rank 43: 23255.78 + rank 44: 23255.74 + rank 45: 23255.76 + rank 46: 23255.77 + rank 47: 23255.79 + rank 48: 23255.23 + rank 49: 23255.19 + rank 50: 23255.24 + rank 51: 23255.20 + rank 52: 23255.20 + rank 53: 23255.18 + rank 54: 23255.19 + rank 55: 23255.19 + rank 56: 23255.36 + rank 57: 23255.38 + rank 58: 23255.38 + rank 59: 23255.38 + rank 60: 23255.42 + rank 61: 23255.38 + rank 62: 23255.36 + rank 63: 23255.39 + [2026-10-11 21:23:01] iteration 121/ 2384 | consumed samples: 3872 | elapsed time per iteration (ms): 220988.3 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.98% | FLOPs estimator: workload-aware | logical tokens/s: 75918.9 | valid tokens/s: 75918.9 | physical tokens/s: 76448.7 | loss tokens/s: 59616.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.536566E-01 | lm-global-supervised-tokens: 1.317460E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.737567E-02 | lm-token-buffer-mib: 2.333553E+02 | global_load_balancing_loss: 1.003921E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11228.936 | params rms: 0.060801 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:54:56.441622 | finish at 2026-10-17 16:17:58 + [2026-10-11 21:26:48] iteration 122/ 2384 | consumed samples: 3904 | elapsed time per iteration (ms): 226557.0 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 74052.8 | valid tokens/s: 74052.8 | physical tokens/s: 74487.4 | loss tokens/s: 50482.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.167017E-01 | lm-global-supervised-tokens: 1.143714E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.056727E-02 | lm-token-buffer-mib: 2.327152E+02 | global_load_balancing_loss: 1.000908E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11228.900 | params rms: 0.060801 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:21:12.009326 | finish at 2026-10-17 19:48:00 + [2026-10-11 21:31:38] iteration 123/ 2384 | consumed samples: 3936 | elapsed time per iteration (ms): 290075.4 | throughput per GPU (TFLOP/s/GPU): 17.2 | MFU 5.51% | FLOPs estimator: workload-aware | logical tokens/s: 57837.3 | valid tokens/s: 57837.3 | physical tokens/s: 58574.6 | loss tokens/s: 42608.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.758559E-01 | lm-global-supervised-tokens: 1.235969E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.194585E-02 | lm-token-buffer-mib: 2.369176E+02 | global_load_balancing_loss: 1.000974E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11228.861 | params rms: 0.060801 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 14:11:00.504994 | finish at 2026-10-19 11:42:38 + [2026-10-11 21:35:18] iteration 124/ 2384 | consumed samples: 3968 | elapsed time per iteration (ms): 219724.5 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 76355.6 | valid tokens/s: 76355.6 | physical tokens/s: 76753.9 | loss tokens/s: 56210.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.528232E-01 | lm-global-supervised-tokens: 1.235078E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.090060E-02 | lm-token-buffer-mib: 2.336294E+02 | global_load_balancing_loss: 9.998223E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11228.821 | params rms: 0.060800 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:56:17.365556 | finish at 2026-10-17 15:31:35 + [2026-10-11 21:39:03] iteration 125/ 2384 | consumed samples: 4000 | elapsed time per iteration (ms): 225100.5 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 74532.1 | valid tokens/s: 74532.1 | physical tokens/s: 75682.2 | loss tokens/s: 57388.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.403578E-01 | lm-global-supervised-tokens: 1.291810E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.427496E-02 | lm-token-buffer-mib: 2.362575E+02 | global_load_balancing_loss: 1.000776E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11228.779 | params rms: 0.060800 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:15:01.927410 | finish at 2026-10-17 18:54:05 + [2026-10-11 21:42:45] iteration 126/ 2384 | consumed samples: 4032 | elapsed time per iteration (ms): 222498.5 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 75403.6 | valid tokens/s: 75403.6 | physical tokens/s: 75836.1 | loss tokens/s: 57489.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.903303E-01 | lm-global-supervised-tokens: 1.279142E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.577711E-02 | lm-token-buffer-mib: 2.323981E+02 | global_load_balancing_loss: 9.997349E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11228.738 | params rms: 0.060800 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:33:21.570178 | finish at 2026-10-17 17:16:07 + [2026-10-11 21:46:28] iteration 127/ 2384 | consumed samples: 4064 | elapsed time per iteration (ms): 222965.7 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 75245.6 | valid tokens/s: 75245.6 | physical tokens/s: 75864.3 | loss tokens/s: 55749.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.376373E-01 | lm-global-supervised-tokens: 1.243031E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.808980E-02 | lm-token-buffer-mib: 2.347729E+02 | global_load_balancing_loss: 1.002044E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11228.696 | params rms: 0.060800 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:47:13.634206 | finish at 2026-10-17 17:33:42 + [2026-10-11 21:50:10] iteration 128/ 2384 | consumed samples: 4096 | elapsed time per iteration (ms): 222121.7 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 75531.5 | valid tokens/s: 75531.5 | physical tokens/s: 76645.0 | loss tokens/s: 62568.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.772873E-01 | lm-global-supervised-tokens: 1.389789E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.345177E-02 | lm-token-buffer-mib: 2.362272E+02 | global_load_balancing_loss: 1.001541E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11228.654 | params rms: 0.060800 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:11:46.638142 | finish at 2026-10-17 17:01:57 + [2026-10-11 21:53:54] iteration 129/ 2384 | consumed samples: 4128 | elapsed time per iteration (ms): 224070.5 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 74874.5 | valid tokens/s: 74874.5 | physical tokens/s: 75507.4 | loss tokens/s: 55142.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.787602E-01 | lm-global-supervised-tokens: 1.235578E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.659599E-02 | lm-token-buffer-mib: 2.342467E+02 | global_load_balancing_loss: 9.999542E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11228.608 | params rms: 0.060799 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:21:18.946623 | finish at 2026-10-17 18:15:13 + [2026-10-11 21:58:01] iteration 130/ 2384 | consumed samples: 4160 | elapsed time per iteration (ms): 246344.4 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 68104.6 | valid tokens/s: 68104.6 | physical tokens/s: 68867.0 | loss tokens/s: 54293.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.577134E-01 | lm-global-supervised-tokens: 1.337492E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.870843E-02 | lm-token-buffer-mib: 2.344940E+02 | global_load_balancing_loss: 1.002691E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11228.556 | params rms: 0.060799 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:14:20.360736 | finish at 2026-10-18 08:12:21 + [2026-10-11 22:01:43] iteration 131/ 2384 | consumed samples: 4192 | elapsed time per iteration (ms): 222062.5 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 75551.6 | valid tokens/s: 75551.6 | physical tokens/s: 76278.9 | loss tokens/s: 58287.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.018768E-01 | lm-global-supervised-tokens: 1.294336E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.447701E-02 | lm-token-buffer-mib: 2.352457E+02 | global_load_balancing_loss: 1.001520E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11228.502 | params rms: 0.060799 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:58:26.712052 | finish at 2026-10-17 17:00:10 + [2026-10-11 22:05:23] iteration 132/ 2384 | consumed samples: 4224 | elapsed time per iteration (ms): 219929.4 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 76284.3 | valid tokens/s: 76284.3 | physical tokens/s: 76741.2 | loss tokens/s: 57206.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.751138E-01 | lm-global-supervised-tokens: 1.258133E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.847095E-02 | lm-token-buffer-mib: 2.338000E+02 | global_load_balancing_loss: 1.001266E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11228.447 | params rms: 0.060798 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:34:40.908859 | finish at 2026-10-17 15:40:04 + [2026-10-11 22:09:06] iteration 133/ 2384 | consumed samples: 4256 | elapsed time per iteration (ms): 222904.5 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75266.3 | valid tokens/s: 75266.3 | physical tokens/s: 75931.6 | loss tokens/s: 58258.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.148952E-01 | lm-global-supervised-tokens: 1.298598E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.598614E-02 | lm-token-buffer-mib: 2.336385E+02 | global_load_balancing_loss: 1.001104E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11228.388 | params rms: 0.060798 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:22:37.996780 | finish at 2026-10-17 17:31:44 + [2026-10-11 22:12:47] iteration 134/ 2384 | consumed samples: 4288 | elapsed time per iteration (ms): 221339.8 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 75798.4 | valid tokens/s: 75798.4 | physical tokens/s: 76390.9 | loss tokens/s: 58328.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.921655E-01 | lm-global-supervised-tokens: 1.291036E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.425901E-02 | lm-token-buffer-mib: 2.336734E+02 | global_load_balancing_loss: 1.001029E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11228.325 | params rms: 0.060798 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:20:14.660776 | finish at 2026-10-17 16:33:02 + [2026-10-11 22:16:29] iteration 135/ 2384 | consumed samples: 4320 | elapsed time per iteration (ms): 222429.8 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 75426.8 | valid tokens/s: 75426.8 | physical tokens/s: 76394.7 | loss tokens/s: 55663.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.090074E-01 | lm-global-supervised-tokens: 1.238121E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.777442E-02 | lm-token-buffer-mib: 2.339950E+02 | global_load_balancing_loss: 9.998091E-01 | loss scale: 1.0 | grad norm: 0.200 | params norm: 11228.259 | params rms: 0.060797 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:57:24.554322 | finish at 2026-10-17 17:13:54 + [2026-10-11 22:20:17] iteration 136/ 2384 | consumed samples: 4352 | elapsed time per iteration (ms): 227572.1 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 73722.4 | valid tokens/s: 73722.4 | physical tokens/s: 74593.1 | loss tokens/s: 54965.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.206324E-01 | lm-global-supervised-tokens: 1.250870E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.216415E-02 | lm-token-buffer-mib: 2.363593E+02 | global_load_balancing_loss: 1.000884E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11228.188 | params rms: 0.060797 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:06:22.034000 | finish at 2026-10-17 20:26:39 + [2026-10-11 22:24:06] iteration 137/ 2384 | consumed samples: 4384 | elapsed time per iteration (ms): 229434.9 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 73123.8 | valid tokens/s: 73123.8 | physical tokens/s: 74235.4 | loss tokens/s: 55031.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.649649E-01 | lm-global-supervised-tokens: 1.262608E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.489211E-02 | lm-token-buffer-mib: 2.378616E+02 | global_load_balancing_loss: 1.000284E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11228.116 | params rms: 0.060797 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:12:20.312011 | finish at 2026-10-17 21:36:27 + [2026-10-11 22:27:48] iteration 138/ 2384 | consumed samples: 4416 | elapsed time per iteration (ms): 221821.1 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 75634.0 | valid tokens/s: 75634.0 | physical tokens/s: 76625.1 | loss tokens/s: 57867.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.431332E-01 | lm-global-supervised-tokens: 1.283632E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.529922E-02 | lm-token-buffer-mib: 2.359857E+02 | global_load_balancing_loss: 9.998876E-01 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11228.042 | params rms: 0.060796 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:23:30.089385 | finish at 2026-10-17 16:51:18 + [2026-10-11 22:31:32] iteration 139/ 2384 | consumed samples: 4448 | elapsed time per iteration (ms): 223356.9 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 75113.8 | valid tokens/s: 75113.8 | physical tokens/s: 75733.1 | loss tokens/s: 56544.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.202732E-01 | lm-global-supervised-tokens: 1.262970E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.980568E-02 | lm-token-buffer-mib: 2.339857E+02 | global_load_balancing_loss: 1.000968E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11227.968 | params rms: 0.060796 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:17:16.336793 | finish at 2026-10-17 17:48:48 + [2026-10-11 22:35:34] iteration 140/ 2384 | consumed samples: 4480 | elapsed time per iteration (ms): 242178.9 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.04% | FLOPs estimator: workload-aware | logical tokens/s: 69276.1 | valid tokens/s: 69276.0 | physical tokens/s: 69956.6 | loss tokens/s: 52730.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.698910E-01 | lm-global-supervised-tokens: 1.277015E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.493832E-02 | lm-token-buffer-mib: 2.331728E+02 | global_load_balancing_loss: 9.993447E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11227.892 | params rms: 0.060795 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:57:29.356911 | finish at 2026-10-18 05:33:03 + [2026-10-11 22:39:13] iteration 141/ 2384 | consumed samples: 4512 | elapsed time per iteration (ms): 219240.6 | throughput per GPU (TFLOP/s/GPU): 17.7 | MFU 5.66% | FLOPs estimator: workload-aware | logical tokens/s: 76524.2 | valid tokens/s: 76524.2 | physical tokens/s: 77350.7 | loss tokens/s: 59721.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.773452E-01 | lm-global-supervised-tokens: 1.309334E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.399347E-02 | lm-token-buffer-mib: 2.336782E+02 | global_load_balancing_loss: 9.996161E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11227.816 | params rms: 0.060795 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:35:56.561229 | finish at 2026-10-17 15:15:10 + [2026-10-11 22:42:57] iteration 142/ 2384 | consumed samples: 4544 | elapsed time per iteration (ms): 223515.7 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75060.4 | valid tokens/s: 75060.4 | physical tokens/s: 75807.6 | loss tokens/s: 56788.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.230051E-01 | lm-global-supervised-tokens: 1.269319E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.434761E-02 | lm-token-buffer-mib: 2.357932E+02 | global_load_balancing_loss: 1.001180E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11227.738 | params rms: 0.060795 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:12:02.261100 | finish at 2026-10-17 17:54:59 + [2026-10-11 22:46:42] iteration 143/ 2384 | consumed samples: 4576 | elapsed time per iteration (ms): 225307.1 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 74463.7 | valid tokens/s: 74463.7 | physical tokens/s: 75337.8 | loss tokens/s: 55655.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.283559E-01 | lm-global-supervised-tokens: 1.253959E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.442212E-02 | lm-token-buffer-mib: 2.345326E+02 | global_load_balancing_loss: 1.000391E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11227.659 | params rms: 0.060794 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:15:13.276947 | finish at 2026-10-17 19:01:55 + [2026-10-11 22:50:27] iteration 144/ 2384 | consumed samples: 4608 | elapsed time per iteration (ms): 225326.1 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 74457.4 | valid tokens/s: 74457.4 | physical tokens/s: 75355.3 | loss tokens/s: 51283.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.630206E-01 | lm-global-supervised-tokens: 1.155545E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.804748E-02 | lm-token-buffer-mib: 2.364951E+02 | global_load_balancing_loss: 1.001469E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11227.582 | params rms: 0.060794 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:12:10.447693 | finish at 2026-10-17 19:02:38 + [2026-10-11 22:54:08] iteration 145/ 2384 | consumed samples: 4640 | elapsed time per iteration (ms): 221118.2 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 75874.4 | valid tokens/s: 75874.4 | physical tokens/s: 76265.8 | loss tokens/s: 54095.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.653474E-01 | lm-global-supervised-tokens: 1.196156E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.382828E-02 | lm-token-buffer-mib: 2.323440E+02 | global_load_balancing_loss: 1.000574E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11227.504 | params rms: 0.060793 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:31:23.538908 | finish at 2026-10-17 16:25:32 + [2026-10-11 22:57:49] iteration 146/ 2384 | consumed samples: 4672 | elapsed time per iteration (ms): 221064.6 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 75892.7 | valid tokens/s: 75892.7 | physical tokens/s: 76495.0 | loss tokens/s: 53083.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.023456E-01 | lm-global-supervised-tokens: 1.173493E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.166739E-02 | lm-token-buffer-mib: 2.339254E+02 | global_load_balancing_loss: 1.000652E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11227.423 | params rms: 0.060793 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:25:42.479802 | finish at 2026-10-17 16:23:32 + [2026-10-11 23:01:30] iteration 147/ 2384 | consumed samples: 4704 | elapsed time per iteration (ms): 220843.2 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 75968.7 | valid tokens/s: 75968.7 | physical tokens/s: 76720.6 | loss tokens/s: 52950.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.603686E-01 | lm-global-supervised-tokens: 1.169378E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.982428E-02 | lm-token-buffer-mib: 2.340462E+02 | global_load_balancing_loss: 1.000522E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11227.338 | params rms: 0.060792 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:13:46.236729 | finish at 2026-10-17 16:15:16 + [2026-10-11 23:05:17] iteration 148/ 2384 | consumed samples: 4736 | elapsed time per iteration (ms): 226901.1 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 73940.7 | valid tokens/s: 73940.7 | physical tokens/s: 74184.6 | loss tokens/s: 53287.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.322488E-01 | lm-global-supervised-tokens: 1.209097E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.569242E-02 | lm-token-buffer-mib: 2.329827E+02 | global_load_balancing_loss: 1.001937E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11227.255 | params rms: 0.060792 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:55:50.803979 | finish at 2026-10-17 20:01:08 + [2026-10-11 23:08:59] iteration 149/ 2384 | consumed samples: 4768 | elapsed time per iteration (ms): 222139.7 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 75525.3 | valid tokens/s: 75525.3 | physical tokens/s: 76075.5 | loss tokens/s: 56346.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.950271E-01 | lm-global-supervised-tokens: 1.251672E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.359681E-02 | lm-token-buffer-mib: 2.326213E+02 | global_load_balancing_loss: 1.001508E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11227.172 | params rms: 0.060791 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:54:42.197918 | finish at 2026-10-17 17:03:41 + [2026-10-11 23:13:01] iteration 150/ 2384 | consumed samples: 4800 | elapsed time per iteration (ms): 241489.4 | throughput per GPU (TFLOP/s/GPU): 17.1 | MFU 5.47% | FLOPs estimator: workload-aware | logical tokens/s: 69473.8 | valid tokens/s: 69473.8 | physical tokens/s: 69984.9 | loss tokens/s: 55445.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.290520E-01 | lm-global-supervised-tokens: 1.338950E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.174157E-02 | lm-token-buffer-mib: 2.338004E+02 | global_load_balancing_loss: 1.000286E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11227.093 | params rms: 0.060791 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:51:27.369998 | finish at 2026-10-18 05:04:28 + [2026-10-11 23:16:47] iteration 151/ 2384 | consumed samples: 4832 | elapsed time per iteration (ms): 226345.9 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 74121.9 | valid tokens/s: 74121.9 | physical tokens/s: 74998.8 | loss tokens/s: 51947.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.466212E-01 | lm-global-supervised-tokens: 1.175807E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.531005E-02 | lm-token-buffer-mib: 2.336354E+02 | global_load_balancing_loss: 1.000913E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11227.016 | params rms: 0.060791 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:23:50.385248 | finish at 2026-10-17 19:40:37 + [2026-10-11 23:20:29] iteration 152/ 2384 | consumed samples: 4864 | elapsed time per iteration (ms): 221782.3 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75647.2 | valid tokens/s: 75647.2 | physical tokens/s: 76152.5 | loss tokens/s: 56246.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.769485E-01 | lm-global-supervised-tokens: 1.247454E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.361975E-02 | lm-token-buffer-mib: 2.331836E+02 | global_load_balancing_loss: 1.000463E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11226.939 | params rms: 0.060790 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:30:17.998335 | finish at 2026-10-17 16:50:47 + [2026-10-11 23:24:13] iteration 153/ 2384 | consumed samples: 4896 | elapsed time per iteration (ms): 224088.7 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 74868.6 | valid tokens/s: 74868.6 | physical tokens/s: 75694.6 | loss tokens/s: 58405.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.968359E-01 | lm-global-supervised-tokens: 1.308794E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.139637E-02 | lm-token-buffer-mib: 2.350981E+02 | global_load_balancing_loss: 1.001056E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11226.863 | params rms: 0.060790 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:52:21.787166 | finish at 2026-10-17 18:16:35 + [2026-10-11 23:27:58] iteration 154/ 2384 | consumed samples: 4928 | elapsed time per iteration (ms): 224725.0 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 74656.5 | valid tokens/s: 74656.5 | physical tokens/s: 75337.6 | loss tokens/s: 55736.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.057441E-01 | lm-global-supervised-tokens: 1.252541E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.009839E-02 | lm-token-buffer-mib: 2.351440E+02 | global_load_balancing_loss: 9.999186E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11226.785 | params rms: 0.060789 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:12:16.684923 | finish at 2026-10-17 18:40:14 + [2026-10-11 23:31:39] iteration 155/ 2384 | consumed samples: 4960 | elapsed time per iteration (ms): 221155.6 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 75861.5 | valid tokens/s: 75861.5 | physical tokens/s: 76448.2 | loss tokens/s: 56978.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.887991E-01 | lm-global-supervised-tokens: 1.260106E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.758954E-02 | lm-token-buffer-mib: 2.353700E+02 | global_load_balancing_loss: 1.001660E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11226.709 | params rms: 0.060789 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:55:55.805986 | finish at 2026-10-17 16:27:35 + [2026-10-11 23:35:24] iteration 156/ 2384 | consumed samples: 4992 | elapsed time per iteration (ms): 224641.9 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 74684.1 | valid tokens/s: 74684.1 | physical tokens/s: 75684.3 | loss tokens/s: 55829.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.384887E-01 | lm-global-supervised-tokens: 1.254170E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.545674E-02 | lm-token-buffer-mib: 2.362814E+02 | global_load_balancing_loss: 1.001913E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11226.634 | params rms: 0.060789 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:01:42.209646 | finish at 2026-10-17 18:37:06 + [2026-10-11 23:39:08] iteration 157/ 2384 | consumed samples: 5024 | elapsed time per iteration (ms): 224960.0 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.41% | FLOPs estimator: workload-aware | logical tokens/s: 74578.4 | valid tokens/s: 74578.4 | physical tokens/s: 75120.3 | loss tokens/s: 57756.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.784125E-01 | lm-global-supervised-tokens: 1.299289E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.707153E-02 | lm-token-buffer-mib: 2.342069E+02 | global_load_balancing_loss: 1.001972E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11226.558 | params rms: 0.060788 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:09:45.959376 | finish at 2026-10-17 18:48:54 + [2026-10-11 23:42:56] iteration 158/ 2384 | consumed samples: 5056 | elapsed time per iteration (ms): 227439.5 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 73765.5 | valid tokens/s: 73765.5 | physical tokens/s: 74537.8 | loss tokens/s: 55250.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.116470E-01 | lm-global-supervised-tokens: 1.256606E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.231850E-02 | lm-token-buffer-mib: 2.337899E+02 | global_load_balancing_loss: 1.000653E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11226.481 | params rms: 0.060788 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:38:00.328800 | finish at 2026-10-17 20:20:56 + [2026-10-11 23:46:39] iteration 159/ 2384 | consumed samples: 5088 | elapsed time per iteration (ms): 223468.7 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75076.2 | valid tokens/s: 75076.2 | physical tokens/s: 75527.7 | loss tokens/s: 56736.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.556369E-01 | lm-global-supervised-tokens: 1.267871E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.420657E-02 | lm-token-buffer-mib: 2.333925E+02 | global_load_balancing_loss: 9.993512E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11226.408 | params rms: 0.060787 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:06:57.843556 | finish at 2026-10-17 17:53:37 + [2026-10-11 23:50:42] iteration 160/ 2384 | consumed samples: 5120 | elapsed time per iteration (ms): 242862.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 69080.9 | valid tokens/s: 69080.9 | physical tokens/s: 69862.6 | loss tokens/s: 53645.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.220375E-01 | lm-global-supervised-tokens: 1.302835E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.916016E-02 | lm-token-buffer-mib: 2.340170E+02 | global_load_balancing_loss: 1.000098E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11226.335 | params rms: 0.060787 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:02:05.538151 | finish at 2026-10-18 05:52:48 +times across ranks (ms): + save-checkpoint: + rank 0: 25676.92 + rank 1: 25676.89 + rank 2: 25676.87 + rank 3: 25676.78 + rank 4: 25676.87 + rank 5: 25676.90 + rank 6: 25676.90 + rank 7: 25676.90 + rank 8: 25676.96 + rank 9: 25676.99 + rank 10: 25677.04 + rank 11: 25676.98 + rank 12: 25676.97 + rank 13: 25676.95 + rank 14: 25676.97 + rank 15: 25676.97 + rank 16: 25676.60 + rank 17: 25676.59 + rank 18: 25676.62 + rank 19: 25676.60 + rank 20: 25676.61 + rank 21: 25676.59 + rank 22: 25676.62 + rank 23: 25676.60 + rank 24: 25677.16 + rank 25: 25677.11 + rank 26: 25677.09 + rank 27: 25677.11 + rank 28: 25677.06 + rank 29: 25677.12 + rank 30: 25677.11 + rank 31: 25676.65 + rank 32: 25676.58 + rank 33: 25676.52 + rank 34: 25676.57 + rank 35: 25676.61 + rank 36: 25676.54 + rank 37: 25676.54 + rank 38: 25676.51 + rank 39: 25676.51 + rank 40: 25676.81 + rank 41: 25676.80 + rank 42: 25676.78 + rank 43: 25676.80 + rank 44: 25676.78 + rank 45: 25676.77 + rank 46: 25676.79 + rank 47: 25676.81 + rank 48: 25676.99 + rank 49: 25677.01 + rank 50: 25677.13 + rank 51: 25677.09 + rank 52: 25677.04 + rank 53: 25676.89 + rank 54: 25677.07 + rank 55: 25676.84 + rank 56: 25676.61 + rank 57: 25676.66 + rank 58: 25676.66 + rank 59: 25676.65 + rank 60: 25676.64 + rank 61: 25676.56 + rank 62: 25676.60 + rank 63: 25676.58 + [2026-10-11 23:54:54] iteration 161/ 2384 | consumed samples: 5152 | elapsed time per iteration (ms): 225944.5 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.23% | FLOPs estimator: workload-aware | logical tokens/s: 74253.6 | valid tokens/s: 74253.6 | physical tokens/s: 74935.8 | loss tokens/s: 58479.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.852767E-01 | lm-global-supervised-tokens: 1.321301E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.837313E-01 | lm-token-buffer-mib: 2.329227E+02 | global_load_balancing_loss: 9.994587E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11226.262 | params rms: 0.060787 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:31:14.632972 | finish at 2026-10-17 19:26:09 + [2026-10-11 23:58:44] iteration 162/ 2384 | consumed samples: 5184 | elapsed time per iteration (ms): 230281.7 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 72855.0 | valid tokens/s: 72855.0 | physical tokens/s: 73754.3 | loss tokens/s: 53799.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.031324E-01 | lm-global-supervised-tokens: 1.238893E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.178726E-02 | lm-token-buffer-mib: 2.360675E+02 | global_load_balancing_loss: 1.000865E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11226.189 | params rms: 0.060786 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:08:05.980080 | finish at 2026-10-17 22:06:50 + [2026-10-12 00:02:26] iteration 163/ 2384 | consumed samples: 5216 | elapsed time per iteration (ms): 222084.0 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 75544.4 | valid tokens/s: 75544.4 | physical tokens/s: 76126.6 | loss tokens/s: 58377.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.447223E-01 | lm-global-supervised-tokens: 1.296461E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.686892E-02 | lm-token-buffer-mib: 2.324132E+02 | global_load_balancing_loss: 1.001694E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11226.114 | params rms: 0.060786 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:00:48.655854 | finish at 2026-10-17 17:03:15 + [2026-10-12 00:06:07] iteration 164/ 2384 | consumed samples: 5248 | elapsed time per iteration (ms): 220642.6 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.52% | FLOPs estimator: workload-aware | logical tokens/s: 76037.8 | valid tokens/s: 76037.8 | physical tokens/s: 76569.1 | loss tokens/s: 61253.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.297733E-01 | lm-global-supervised-tokens: 1.351514E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.632008E-02 | lm-token-buffer-mib: 2.335800E+02 | global_load_balancing_loss: 9.998623E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11226.042 | params rms: 0.060785 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:03:46.664758 | finish at 2026-10-17 16:09:54 + [2026-10-12 00:09:49] iteration 165/ 2384 | consumed samples: 5280 | elapsed time per iteration (ms): 222130.0 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 75528.7 | valid tokens/s: 75528.7 | physical tokens/s: 76456.8 | loss tokens/s: 58869.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.376346E-01 | lm-global-supervised-tokens: 1.307663E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.287194E-02 | lm-token-buffer-mib: 2.357268E+02 | global_load_balancing_loss: 1.000108E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11225.970 | params rms: 0.060785 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:55:06.513107 | finish at 2026-10-17 17:04:56 + [2026-10-12 00:13:34] iteration 166/ 2384 | consumed samples: 5312 | elapsed time per iteration (ms): 224806.2 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 74629.5 | valid tokens/s: 74629.5 | physical tokens/s: 75197.6 | loss tokens/s: 57829.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.055957E-01 | lm-global-supervised-tokens: 1.300043E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.855336E-02 | lm-token-buffer-mib: 2.332480E+02 | global_load_balancing_loss: 1.001221E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11225.894 | params rms: 0.060785 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:30:20.217762 | finish at 2026-10-17 18:43:54 + [2026-10-12 00:17:14] iteration 167/ 2384 | consumed samples: 5344 | elapsed time per iteration (ms): 220506.1 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 76084.9 | valid tokens/s: 76084.9 | physical tokens/s: 76883.2 | loss tokens/s: 56254.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.531193E-01 | lm-global-supervised-tokens: 1.240451E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.623668E-02 | lm-token-buffer-mib: 2.332529E+02 | global_load_balancing_loss: 1.000437E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11225.821 | params rms: 0.060784 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:47:42.008765 | finish at 2026-10-17 16:04:56 + [2026-10-12 00:20:56] iteration 168/ 2384 | consumed samples: 5376 | elapsed time per iteration (ms): 221616.7 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75703.6 | valid tokens/s: 75703.6 | physical tokens/s: 76615.4 | loss tokens/s: 53906.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.110702E-01 | lm-global-supervised-tokens: 1.194655E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.955727E-02 | lm-token-buffer-mib: 2.344926E+02 | global_load_balancing_loss: 1.000183E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11225.748 | params rms: 0.060784 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:25:02.702154 | finish at 2026-10-17 16:45:59 + [2026-10-12 00:24:41] iteration 169/ 2384 | consumed samples: 5408 | elapsed time per iteration (ms): 224610.8 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 74694.6 | valid tokens/s: 74694.6 | physical tokens/s: 75937.8 | loss tokens/s: 56470.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.103942E-01 | lm-global-supervised-tokens: 1.268387E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.580772E-02 | lm-token-buffer-mib: 2.370105E+02 | global_load_balancing_loss: 9.996024E-01 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11225.675 | params rms: 0.060783 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:11:52.988074 | finish at 2026-10-17 18:36:34 + [2026-10-12 00:28:46] iteration 170/ 2384 | consumed samples: 5440 | elapsed time per iteration (ms): 245072.9 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 68458.0 | valid tokens/s: 68458.0 | physical tokens/s: 69328.7 | loss tokens/s: 49678.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.090848E-01 | lm-global-supervised-tokens: 1.217486E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.415072E-02 | lm-token-buffer-mib: 2.354704E+02 | global_load_balancing_loss: 1.000550E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11225.601 | params rms: 0.060783 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:43:11.492573 | finish at 2026-10-18 07:11:57 + [2026-10-12 00:32:27] iteration 171/ 2384 | consumed samples: 5472 | elapsed time per iteration (ms): 221183.5 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 75851.9 | valid tokens/s: 75851.9 | physical tokens/s: 76220.9 | loss tokens/s: 58616.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.797973E-01 | lm-global-supervised-tokens: 1.296495E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.100397E-02 | lm-token-buffer-mib: 2.334531E+02 | global_load_balancing_loss: 9.996896E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11225.525 | params rms: 0.060783 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:57:59.156315 | finish at 2026-10-17 16:30:26 + [2026-10-12 00:36:08] iteration 172/ 2384 | consumed samples: 5504 | elapsed time per iteration (ms): 221341.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.16% | FLOPs estimator: workload-aware | logical tokens/s: 75797.8 | valid tokens/s: 75797.8 | physical tokens/s: 76086.2 | loss tokens/s: 56233.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.855377E-01 | lm-global-supervised-tokens: 1.244684E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.198069E-02 | lm-token-buffer-mib: 2.333381E+02 | global_load_balancing_loss: 1.000795E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11225.450 | params rms: 0.060782 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:00:07.329537 | finish at 2026-10-17 16:36:15 + [2026-10-12 00:39:48] iteration 173/ 2384 | consumed samples: 5536 | elapsed time per iteration (ms): 219837.7 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 76316.2 | valid tokens/s: 76316.2 | physical tokens/s: 76864.3 | loss tokens/s: 58163.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.566285E-01 | lm-global-supervised-tokens: 1.278647E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.374965E-02 | lm-token-buffer-mib: 2.342867E+02 | global_load_balancing_loss: 1.001073E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11225.374 | params rms: 0.060782 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:01:01.221404 | finish at 2026-10-17 15:40:49 + [2026-10-12 00:43:31] iteration 174/ 2384 | consumed samples: 5568 | elapsed time per iteration (ms): 222829.0 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 75291.7 | valid tokens/s: 75291.7 | physical tokens/s: 76388.2 | loss tokens/s: 60420.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.714797E-01 | lm-global-supervised-tokens: 1.346344E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.033878E-02 | lm-token-buffer-mib: 2.335940E+02 | global_load_balancing_loss: 1.002322E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11225.293 | params rms: 0.060781 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:47:32.125821 | finish at 2026-10-17 17:31:03 + [2026-10-12 00:47:15] iteration 175/ 2384 | consumed samples: 5600 | elapsed time per iteration (ms): 224477.0 | throughput per GPU (TFLOP/s/GPU): 18.3 | MFU 5.86% | FLOPs estimator: workload-aware | logical tokens/s: 74739.0 | valid tokens/s: 74739.0 | physical tokens/s: 75742.4 | loss tokens/s: 58614.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.167377E-01 | lm-global-supervised-tokens: 1.315760E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.494174E-02 | lm-token-buffer-mib: 2.330754E+02 | global_load_balancing_loss: 1.000387E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11225.216 | params rms: 0.060781 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:44:29.588716 | finish at 2026-10-17 18:31:45 + [2026-10-12 00:51:01] iteration 176/ 2384 | consumed samples: 5632 | elapsed time per iteration (ms): 226062.5 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 74214.7 | valid tokens/s: 74214.7 | physical tokens/s: 74907.4 | loss tokens/s: 53621.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.407216E-01 | lm-global-supervised-tokens: 1.212184E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.252905E-02 | lm-token-buffer-mib: 2.332556E+02 | global_load_balancing_loss: 1.002494E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11225.142 | params rms: 0.060780 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:39:05.984207 | finish at 2026-10-17 19:30:07 + [2026-10-12 00:54:43] iteration 177/ 2384 | consumed samples: 5664 | elapsed time per iteration (ms): 221806.6 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75638.8 | valid tokens/s: 75638.8 | physical tokens/s: 76414.0 | loss tokens/s: 57666.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.723358E-01 | lm-global-supervised-tokens: 1.279072E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.796028E-02 | lm-token-buffer-mib: 2.333643E+02 | global_load_balancing_loss: 1.000134E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11225.072 | params rms: 0.060780 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:58:47.166407 | finish at 2026-10-17 16:53:30 + [2026-10-12 00:58:23] iteration 178/ 2384 | consumed samples: 5696 | elapsed time per iteration (ms): 219655.9 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.33% | FLOPs estimator: workload-aware | logical tokens/s: 76379.3 | valid tokens/s: 76379.3 | physical tokens/s: 77042.1 | loss tokens/s: 55383.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.205460E-01 | lm-global-supervised-tokens: 1.216534E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.357751E-02 | lm-token-buffer-mib: 2.327698E+02 | global_load_balancing_loss: 9.992905E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11225.005 | params rms: 0.060780 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:36:00.831777 | finish at 2026-10-17 15:34:24 + [2026-10-12 01:02:05] iteration 179/ 2384 | consumed samples: 5728 | elapsed time per iteration (ms): 222556.8 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 75383.9 | valid tokens/s: 75383.9 | physical tokens/s: 76026.7 | loss tokens/s: 57233.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.987313E-01 | lm-global-supervised-tokens: 1.273778E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.269511E-02 | lm-token-buffer-mib: 2.337025E+02 | global_load_balancing_loss: 9.992995E-01 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11224.941 | params rms: 0.060779 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:18:57.780554 | finish at 2026-10-17 17:21:03 + [2026-10-12 01:06:09] iteration 180/ 2384 | consumed samples: 5760 | elapsed time per iteration (ms): 243405.4 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.16% | FLOPs estimator: workload-aware | logical tokens/s: 68926.9 | valid tokens/s: 68926.9 | physical tokens/s: 69742.9 | loss tokens/s: 53136.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.126995E-01 | lm-global-supervised-tokens: 1.293370E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.745189E-02 | lm-token-buffer-mib: 2.347305E+02 | global_load_balancing_loss: 9.987878E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11224.875 | params rms: 0.060779 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:01:05.424438 | finish at 2026-10-18 06:07:14 + [2026-10-12 01:09:53] iteration 181/ 2384 | consumed samples: 5792 | elapsed time per iteration (ms): 224171.4 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 74840.8 | valid tokens/s: 74840.8 | physical tokens/s: 76036.1 | loss tokens/s: 58760.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.508860E-01 | lm-global-supervised-tokens: 1.317241E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.490596E-02 | lm-token-buffer-mib: 2.371384E+02 | global_load_balancing_loss: 1.001397E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11224.809 | params rms: 0.060779 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:10:49.518195 | finish at 2026-10-17 18:20:42 + [2026-10-12 01:13:33] iteration 182/ 2384 | consumed samples: 5824 | elapsed time per iteration (ms): 220237.0 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 76178.0 | valid tokens/s: 76178.0 | physical tokens/s: 76859.2 | loss tokens/s: 58780.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.634713E-01 | lm-global-supervised-tokens: 1.294564E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.402731E-02 | lm-token-buffer-mib: 2.352774E+02 | global_load_balancing_loss: 1.001024E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11224.743 | params rms: 0.060778 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:42:41.909675 | finish at 2026-10-17 15:56:15 + [2026-10-12 01:17:12] iteration 183/ 2384 | consumed samples: 5856 | elapsed time per iteration (ms): 219213.6 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 76533.6 | valid tokens/s: 76533.6 | physical tokens/s: 77160.2 | loss tokens/s: 56586.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.017923E-01 | lm-global-supervised-tokens: 1.240448E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.831781E-02 | lm-token-buffer-mib: 2.348916E+02 | global_load_balancing_loss: 1.000229E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11224.677 | params rms: 0.060778 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:01:29.141821 | finish at 2026-10-17 15:18:42 + [2026-10-12 01:20:55] iteration 184/ 2384 | consumed samples: 5888 | elapsed time per iteration (ms): 222229.9 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 75494.8 | valid tokens/s: 75494.8 | physical tokens/s: 76258.9 | loss tokens/s: 56748.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.072042E-01 | lm-global-supervised-tokens: 1.261111E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.357504E-02 | lm-token-buffer-mib: 2.342166E+02 | global_load_balancing_loss: 1.003136E+00 | loss scale: 1.0 | grad norm: 0.279 | params norm: 11224.610 | params rms: 0.060778 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:48:25.749321 | finish at 2026-10-17 17:09:20 + [2026-10-12 01:24:43] iteration 185/ 2384 | consumed samples: 5920 | elapsed time per iteration (ms): 228329.5 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 73478.0 | valid tokens/s: 73478.0 | physical tokens/s: 74724.8 | loss tokens/s: 53840.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.912447E-01 | lm-global-supervised-tokens: 1.229338E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.497845E-02 | lm-token-buffer-mib: 2.391257E+02 | global_load_balancing_loss: 1.002072E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11224.543 | params rms: 0.060777 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:28:16.632277 | finish at 2026-10-17 20:53:00 + [2026-10-12 01:28:25] iteration 186/ 2384 | consumed samples: 5952 | elapsed time per iteration (ms): 222154.9 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 75520.1 | valid tokens/s: 75520.1 | physical tokens/s: 76417.2 | loss tokens/s: 63312.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.822520E-01 | lm-global-supervised-tokens: 1.406520E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.032479E-02 | lm-token-buffer-mib: 2.337745E+02 | global_load_balancing_loss: 1.001919E+00 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11224.478 | params rms: 0.060777 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:38:16.446781 | finish at 2026-10-17 17:06:42 + [2026-10-12 01:32:06] iteration 187/ 2384 | consumed samples: 5984 | elapsed time per iteration (ms): 221052.1 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 75897.0 | valid tokens/s: 75897.0 | physical tokens/s: 76285.7 | loss tokens/s: 58176.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.245247E-01 | lm-global-supervised-tokens: 1.286004E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.137442E-02 | lm-token-buffer-mib: 2.334988E+02 | global_load_balancing_loss: 9.986920E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11224.412 | params rms: 0.060777 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:54:11.471956 | finish at 2026-10-17 16:26:18 + [2026-10-12 01:35:50] iteration 188/ 2384 | consumed samples: 6016 | elapsed time per iteration (ms): 223415.1 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 75094.2 | valid tokens/s: 75094.2 | physical tokens/s: 75503.4 | loss tokens/s: 55009.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.120548E-01 | lm-global-supervised-tokens: 1.228994E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.344971E-02 | lm-token-buffer-mib: 2.334737E+02 | global_load_balancing_loss: 9.975771E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11224.343 | params rms: 0.060776 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:16:59.647250 | finish at 2026-10-17 17:52:49 + [2026-10-12 01:39:32] iteration 189/ 2384 | consumed samples: 6048 | elapsed time per iteration (ms): 222243.2 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 75490.3 | valid tokens/s: 75490.3 | physical tokens/s: 75899.0 | loss tokens/s: 55300.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.026169E-01 | lm-global-supervised-tokens: 1.229026E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.642649E-02 | lm-token-buffer-mib: 2.331660E+02 | global_load_balancing_loss: 9.999895E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11224.270 | params rms: 0.060776 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:30:23.840363 | finish at 2026-10-17 17:09:56 + [2026-10-12 01:43:37] iteration 190/ 2384 | consumed samples: 6080 | elapsed time per iteration (ms): 245558.1 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 68322.8 | valid tokens/s: 68322.8 | physical tokens/s: 68825.0 | loss tokens/s: 54455.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.081380E-01 | lm-global-supervised-tokens: 1.337192E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.382632E-02 | lm-token-buffer-mib: 2.331953E+02 | global_load_balancing_loss: 1.001040E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11224.199 | params rms: 0.060775 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:39:14.375642 | finish at 2026-10-18 07:22:52 + [2026-10-12 01:47:22] iteration 191/ 2384 | consumed samples: 6112 | elapsed time per iteration (ms): 224432.2 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 74753.9 | valid tokens/s: 74753.9 | physical tokens/s: 75379.1 | loss tokens/s: 59878.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.093939E-01 | lm-global-supervised-tokens: 1.343873E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.766610E-02 | lm-token-buffer-mib: 2.340310E+02 | global_load_balancing_loss: 1.000216E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11224.127 | params rms: 0.060775 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:42:59.822344 | finish at 2026-10-17 18:30:22 + [2026-10-12 01:51:04] iteration 192/ 2384 | consumed samples: 6144 | elapsed time per iteration (ms): 222607.8 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 75366.5 | valid tokens/s: 75366.5 | physical tokens/s: 76256.8 | loss tokens/s: 55992.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.910332E-01 | lm-global-supervised-tokens: 1.246441E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.055388E-02 | lm-token-buffer-mib: 2.339784E+02 | global_load_balancing_loss: 1.001927E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11224.055 | params rms: 0.060775 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:32:36.321133 | finish at 2026-10-17 17:23:41 + [2026-10-12 01:54:47] iteration 193/ 2384 | consumed samples: 6176 | elapsed time per iteration (ms): 222935.6 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 75255.6 | valid tokens/s: 75255.6 | physical tokens/s: 75822.9 | loss tokens/s: 49270.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.182750E-01 | lm-global-supervised-tokens: 1.098418E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.330302E-02 | lm-token-buffer-mib: 2.344652E+02 | global_load_balancing_loss: 1.001835E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11223.981 | params rms: 0.060774 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:40:51.796785 | finish at 2026-10-17 17:35:39 + [2026-10-12 01:58:27] iteration 194/ 2384 | consumed samples: 6208 | elapsed time per iteration (ms): 219266.7 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.38% | FLOPs estimator: workload-aware | logical tokens/s: 76515.1 | valid tokens/s: 76515.1 | physical tokens/s: 77178.6 | loss tokens/s: 60904.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.552430E-01 | lm-global-supervised-tokens: 1.335432E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.637481E-02 | lm-token-buffer-mib: 2.334655E+02 | global_load_balancing_loss: 1.001672E+00 | loss scale: 1.0 | grad norm: 0.256 | params norm: 11223.903 | params rms: 0.060774 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:23:14.019806 | finish at 2026-10-17 15:21:41 + [2026-10-12 02:02:09] iteration 195/ 2384 | consumed samples: 6240 | elapsed time per iteration (ms): 221889.4 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.24% | FLOPs estimator: workload-aware | logical tokens/s: 75610.6 | valid tokens/s: 75610.6 | physical tokens/s: 76665.8 | loss tokens/s: 63399.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.334089E-01 | lm-global-supervised-tokens: 1.406761E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.845316E-02 | lm-token-buffer-mib: 2.348499E+02 | global_load_balancing_loss: 1.001364E+00 | loss scale: 1.0 | grad norm: 0.547 | params norm: 11223.830 | params rms: 0.060773 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:55:15.888783 | finish at 2026-10-17 16:57:24 + [2026-10-12 02:05:52] iteration 196/ 2384 | consumed samples: 6272 | elapsed time per iteration (ms): 223378.7 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75106.6 | valid tokens/s: 75106.6 | physical tokens/s: 76303.1 | loss tokens/s: 59090.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.547132E-01 | lm-global-supervised-tokens: 1.319950E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.577043E-02 | lm-token-buffer-mib: 2.360246E+02 | global_load_balancing_loss: 1.000656E+00 | loss scale: 1.0 | grad norm: 0.340 | params norm: 11223.761 | params rms: 0.060773 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:45:52.577381 | finish at 2026-10-17 17:51:44 + [2026-10-12 02:09:36] iteration 197/ 2384 | consumed samples: 6304 | elapsed time per iteration (ms): 223684.2 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75004.0 | valid tokens/s: 75004.0 | physical tokens/s: 75441.2 | loss tokens/s: 56100.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.708198E-01 | lm-global-supervised-tokens: 1.254877E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.284621E-02 | lm-token-buffer-mib: 2.332295E+02 | global_load_balancing_loss: 1.000383E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11223.693 | params rms: 0.060773 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:53:17.336642 | finish at 2026-10-17 18:02:53 + [2026-10-12 02:13:16] iteration 198/ 2384 | consumed samples: 6336 | elapsed time per iteration (ms): 220040.6 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.98% | FLOPs estimator: workload-aware | logical tokens/s: 76245.8 | valid tokens/s: 76245.8 | physical tokens/s: 77008.1 | loss tokens/s: 58230.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.174424E-01 | lm-global-supervised-tokens: 1.281307E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.481799E-02 | lm-token-buffer-mib: 2.329339E+02 | global_load_balancing_loss: 1.001467E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11223.626 | params rms: 0.060772 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:36:48.645413 | finish at 2026-10-17 15:50:04 + [2026-10-12 02:17:01] iteration 199/ 2384 | consumed samples: 6368 | elapsed time per iteration (ms): 225362.7 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 74445.4 | valid tokens/s: 74445.4 | physical tokens/s: 75254.5 | loss tokens/s: 57743.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.725706E-01 | lm-global-supervised-tokens: 1.301315E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.621228E-02 | lm-token-buffer-mib: 2.363530E+02 | global_load_balancing_loss: 1.002725E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11223.559 | params rms: 0.060772 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:46:57.449979 | finish at 2026-10-17 19:03:58 + [2026-10-12 02:21:01] iteration 200/ 2384 | consumed samples: 6400 | elapsed time per iteration (ms): 240485.8 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.02% | FLOPs estimator: workload-aware | logical tokens/s: 69763.8 | valid tokens/s: 69763.8 | physical tokens/s: 70767.2 | loss tokens/s: 53941.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.648332E-01 | lm-global-supervised-tokens: 1.297222E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.832846E-02 | lm-token-buffer-mib: 2.345135E+02 | global_load_balancing_loss: 1.000237E+00 | loss scale: 1.0 | grad norm: 0.248 | params norm: 11223.493 | params rms: 0.060772 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:53:40.955498 | finish at 2026-10-18 04:14:42 +times across ranks (ms): + save-checkpoint: + rank 0: 23273.12 + rank 1: 23273.14 + rank 2: 23273.13 + rank 3: 23273.13 + rank 4: 23273.12 + rank 5: 23273.16 + rank 6: 23273.12 + rank 7: 23273.15 + rank 8: 23273.57 + rank 9: 23273.56 + rank 10: 23273.57 + rank 11: 23273.55 + rank 12: 23273.54 + rank 13: 23273.59 + rank 14: 23273.55 + rank 15: 23273.55 + rank 16: 23273.33 + rank 17: 23273.35 + rank 18: 23273.33 + rank 19: 23273.35 + rank 20: 23273.31 + rank 21: 23273.30 + rank 22: 23273.32 + rank 23: 23273.30 + rank 24: 23273.86 + rank 25: 23273.86 + rank 26: 23273.83 + rank 27: 23273.85 + rank 28: 23273.81 + rank 29: 23273.87 + rank 30: 23274.08 + rank 31: 23273.84 + rank 32: 23272.28 + rank 33: 23272.22 + rank 34: 23272.24 + rank 35: 23272.26 + rank 36: 23272.22 + rank 37: 23272.25 + rank 38: 23272.20 + rank 39: 23272.22 + rank 40: 23272.92 + rank 41: 23272.88 + rank 42: 23272.90 + rank 43: 23272.89 + rank 44: 23272.88 + rank 45: 23272.88 + rank 46: 23272.88 + rank 47: 23272.87 + rank 48: 23273.17 + rank 49: 23273.16 + rank 50: 23273.20 + rank 51: 23273.16 + rank 52: 23273.15 + rank 53: 23273.13 + rank 54: 23273.14 + rank 55: 23273.15 + rank 56: 23273.00 + rank 57: 23273.02 + rank 58: 23273.02 + rank 59: 23273.01 + rank 60: 23273.01 + rank 61: 23273.00 + rank 62: 23273.02 + rank 63: 23273.01 + [2026-10-12 02:25:05] iteration 201/ 2384 | consumed samples: 6432 | elapsed time per iteration (ms): 220002.6 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 76259.2 | valid tokens/s: 76259.2 | physical tokens/s: 76881.7 | loss tokens/s: 57868.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.579491E-01 | lm-global-supervised-tokens: 1.273129E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.767904E-02 | lm-token-buffer-mib: 2.342970E+02 | global_load_balancing_loss: 9.997268E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11223.424 | params rms: 0.060771 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:24:25.674660 | finish at 2026-10-17 15:49:30 + [2026-10-12 02:28:51] iteration 202/ 2384 | consumed samples: 6464 | elapsed time per iteration (ms): 226437.4 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 74091.9 | valid tokens/s: 74091.9 | physical tokens/s: 74399.0 | loss tokens/s: 55807.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.217194E-01 | lm-global-supervised-tokens: 1.263683E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.212635E-02 | lm-token-buffer-mib: 2.331131E+02 | global_load_balancing_loss: 9.996564E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11223.357 | params rms: 0.060771 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:14:46.401822 | finish at 2026-10-17 19:43:38 + [2026-10-12 02:32:31] iteration 203/ 2384 | consumed samples: 6496 | elapsed time per iteration (ms): 219445.9 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 76452.5 | valid tokens/s: 76452.5 | physical tokens/s: 77356.7 | loss tokens/s: 58999.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.917669E-01 | lm-global-supervised-tokens: 1.294710E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.946903E-02 | lm-token-buffer-mib: 2.341862E+02 | global_load_balancing_loss: 9.999141E-01 | loss scale: 1.0 | grad norm: 0.354 | params norm: 11223.291 | params rms: 0.060770 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:56:51.468301 | finish at 2026-10-17 15:29:22 + [2026-10-12 02:36:14] iteration 204/ 2384 | consumed samples: 6528 | elapsed time per iteration (ms): 223120.5 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 75193.5 | valid tokens/s: 75193.5 | physical tokens/s: 75902.3 | loss tokens/s: 59433.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.542060E-01 | lm-global-supervised-tokens: 1.326076E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.323533E-02 | lm-token-buffer-mib: 2.339587E+02 | global_load_balancing_loss: 1.000012E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11223.225 | params rms: 0.060770 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:06:42.654848 | finish at 2026-10-17 17:42:56 + [2026-10-12 02:39:54] iteration 205/ 2384 | consumed samples: 6560 | elapsed time per iteration (ms): 220262.5 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 76169.0 | valid tokens/s: 76169.0 | physical tokens/s: 76610.6 | loss tokens/s: 51806.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.431423E-01 | lm-global-supervised-tokens: 1.141098E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.629769E-02 | lm-token-buffer-mib: 2.334006E+02 | global_load_balancing_loss: 1.001105E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11223.159 | params rms: 0.060770 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:19:11.951757 | finish at 2026-10-17 15:59:06 + [2026-10-12 02:43:35] iteration 206/ 2384 | consumed samples: 6592 | elapsed time per iteration (ms): 221322.4 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75804.2 | valid tokens/s: 75804.2 | physical tokens/s: 76315.4 | loss tokens/s: 58102.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.641169E-01 | lm-global-supervised-tokens: 1.285943E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.559403E-02 | lm-token-buffer-mib: 2.326353E+02 | global_load_balancing_loss: 1.001847E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11223.091 | params rms: 0.060769 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:54:00.236215 | finish at 2026-10-17 16:37:36 + [2026-10-12 02:47:16] iteration 207/ 2384 | consumed samples: 6624 | elapsed time per iteration (ms): 220692.0 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 76020.9 | valid tokens/s: 76020.9 | physical tokens/s: 76737.3 | loss tokens/s: 56188.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.721605E-01 | lm-global-supervised-tokens: 1.240028E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.090949E-02 | lm-token-buffer-mib: 2.341474E+02 | global_load_balancing_loss: 1.000909E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11223.022 | params rms: 0.060769 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:27:26.386230 | finish at 2026-10-17 16:14:42 + [2026-10-12 02:50:59] iteration 208/ 2384 | consumed samples: 6656 | elapsed time per iteration (ms): 223304.1 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 75131.6 | valid tokens/s: 75131.6 | physical tokens/s: 75825.3 | loss tokens/s: 56487.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.566142E-01 | lm-global-supervised-tokens: 1.261396E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.699154E-02 | lm-token-buffer-mib: 2.343209E+02 | global_load_balancing_loss: 1.000773E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11222.953 | params rms: 0.060769 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:58:29.653198 | finish at 2026-10-17 17:49:29 + [2026-10-12 02:54:41] iteration 209/ 2384 | consumed samples: 6688 | elapsed time per iteration (ms): 222058.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 75553.2 | valid tokens/s: 75553.2 | physical tokens/s: 76324.5 | loss tokens/s: 56148.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.522104E-01 | lm-global-supervised-tokens: 1.246830E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.805438E-02 | lm-token-buffer-mib: 2.329965E+02 | global_load_balancing_loss: 1.001541E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11222.886 | params rms: 0.060768 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:09:36.545852 | finish at 2026-10-17 17:04:18 + [2026-10-12 02:58:42] iteration 210/ 2384 | consumed samples: 6720 | elapsed time per iteration (ms): 240958.0 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 69627.0 | valid tokens/s: 69627.0 | physical tokens/s: 70017.6 | loss tokens/s: 55477.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.728306E-01 | lm-global-supervised-tokens: 1.336770E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.477390E-02 | lm-token-buffer-mib: 2.325081E+02 | global_load_balancing_loss: 1.003092E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11222.819 | params rms: 0.060768 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:30:42.645749 | finish at 2026-10-18 04:29:25 + [2026-10-12 03:02:22] iteration 211/ 2384 | consumed samples: 6752 | elapsed time per iteration (ms): 219948.4 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 76277.9 | valid tokens/s: 76277.9 | physical tokens/s: 76926.3 | loss tokens/s: 58164.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.841992E-01 | lm-global-supervised-tokens: 1.279308E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.029427E-02 | lm-token-buffer-mib: 2.330927E+02 | global_load_balancing_loss: 9.988627E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11222.752 | params rms: 0.060768 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:45:47.883606 | finish at 2026-10-17 15:48:10 + [2026-10-12 03:06:06] iteration 212/ 2384 | consumed samples: 6784 | elapsed time per iteration (ms): 223659.5 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.23% | FLOPs estimator: workload-aware | logical tokens/s: 75012.3 | valid tokens/s: 75012.3 | physical tokens/s: 75777.2 | loss tokens/s: 56459.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.991840E-01 | lm-global-supervised-tokens: 1.262773E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.681367E-02 | lm-token-buffer-mib: 2.327670E+02 | global_load_balancing_loss: 9.997126E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11222.685 | params rms: 0.060767 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:56:28.377302 | finish at 2026-10-17 18:02:34 + [2026-10-12 03:09:52] iteration 213/ 2384 | consumed samples: 6816 | elapsed time per iteration (ms): 225629.0 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.98% | FLOPs estimator: workload-aware | logical tokens/s: 74357.4 | valid tokens/s: 74357.4 | physical tokens/s: 74647.1 | loss tokens/s: 52786.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.392075E-01 | lm-global-supervised-tokens: 1.191019E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.435032E-02 | lm-token-buffer-mib: 2.326789E+02 | global_load_balancing_loss: 9.996629E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11222.618 | params rms: 0.060767 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:04:00.577522 | finish at 2026-10-17 19:13:52 + [2026-10-12 03:13:34] iteration 214/ 2384 | consumed samples: 6848 | elapsed time per iteration (ms): 222726.0 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 75326.6 | valid tokens/s: 75326.6 | physical tokens/s: 75759.8 | loss tokens/s: 56662.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.749768E-01 | lm-global-supervised-tokens: 1.262012E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.351684E-02 | lm-token-buffer-mib: 2.329218E+02 | global_load_balancing_loss: 1.000341E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11222.553 | params rms: 0.060766 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:15:15.314093 | finish at 2026-10-17 17:28:50 + [2026-10-12 03:17:19] iteration 215/ 2384 | consumed samples: 6880 | elapsed time per iteration (ms): 224948.6 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.82% | FLOPs estimator: workload-aware | logical tokens/s: 74582.4 | valid tokens/s: 74582.4 | physical tokens/s: 75701.4 | loss tokens/s: 58547.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.798841E-01 | lm-global-supervised-tokens: 1.317017E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.967366E-02 | lm-token-buffer-mib: 2.337851E+02 | global_load_balancing_loss: 1.000380E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11222.487 | params rms: 0.060766 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:31:53.591603 | finish at 2026-10-17 18:49:13 + [2026-10-12 03:22:13] iteration 216/ 2384 | consumed samples: 6912 | elapsed time per iteration (ms): 293629.2 | throughput per GPU (TFLOP/s/GPU): 16.8 | MFU 5.38% | FLOPs estimator: workload-aware | logical tokens/s: 57137.4 | valid tokens/s: 57137.4 | physical tokens/s: 58549.2 | loss tokens/s: 44449.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.740965E-01 | lm-global-supervised-tokens: 1.305161E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.594679E-02 | lm-token-buffer-mib: 2.398241E+02 | global_load_balancing_loss: 9.998822E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11222.420 | params rms: 0.060766 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 8:49:48.188263 | finish at 2026-10-19 12:12:01 + [2026-10-12 03:25:54] iteration 217/ 2384 | consumed samples: 6944 | elapsed time per iteration (ms): 221542.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75729.1 | valid tokens/s: 75729.1 | physical tokens/s: 76083.1 | loss tokens/s: 52968.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.026237E-01 | lm-global-supervised-tokens: 1.173476E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.161502E-02 | lm-token-buffer-mib: 2.334203E+02 | global_load_balancing_loss: 1.001711E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11222.350 | params rms: 0.060765 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:21:21.764697 | finish at 2026-10-17 16:47:16 + [2026-10-12 03:29:40] iteration 218/ 2384 | consumed samples: 6976 | elapsed time per iteration (ms): 225372.1 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.29% | FLOPs estimator: workload-aware | logical tokens/s: 74442.2 | valid tokens/s: 74442.2 | physical tokens/s: 75551.8 | loss tokens/s: 56984.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.186422E-01 | lm-global-supervised-tokens: 1.284260E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.458181E-02 | lm-token-buffer-mib: 2.378692E+02 | global_load_balancing_loss: 1.000917E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11222.286 | params rms: 0.060765 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:35:55.974529 | finish at 2026-10-17 19:05:36 + [2026-10-12 03:33:22] iteration 219/ 2384 | consumed samples: 7008 | elapsed time per iteration (ms): 222474.3 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75411.9 | valid tokens/s: 75411.9 | physical tokens/s: 76427.1 | loss tokens/s: 56793.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.311388E-01 | lm-global-supervised-tokens: 1.263510E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.424050E-02 | lm-token-buffer-mib: 2.341876E+02 | global_load_balancing_loss: 1.000761E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11222.224 | params rms: 0.060765 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:47:36.888206 | finish at 2026-10-17 17:20:59 + [2026-10-12 03:37:25] iteration 220/ 2384 | consumed samples: 7040 | elapsed time per iteration (ms): 242790.7 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 69101.5 | valid tokens/s: 69101.5 | physical tokens/s: 70042.3 | loss tokens/s: 54274.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.816531E-01 | lm-global-supervised-tokens: 1.317744E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.915410E-02 | lm-token-buffer-mib: 2.329748E+02 | global_load_balancing_loss: 1.002398E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11222.160 | params rms: 0.060764 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:56:39.155197 | finish at 2026-10-18 05:34:04 + [2026-10-12 03:41:08] iteration 221/ 2384 | consumed samples: 7072 | elapsed time per iteration (ms): 222735.8 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 75323.3 | valid tokens/s: 75323.3 | physical tokens/s: 76031.4 | loss tokens/s: 60734.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.628544E-01 | lm-global-supervised-tokens: 1.352783E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.748620E-02 | lm-token-buffer-mib: 2.337427E+02 | global_load_balancing_loss: 1.001939E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11222.100 | params rms: 0.060764 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:49:37.578263 | finish at 2026-10-17 17:30:45 + [2026-10-12 03:44:52] iteration 222/ 2384 | consumed samples: 7104 | elapsed time per iteration (ms): 223779.1 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 74972.1 | valid tokens/s: 74972.1 | physical tokens/s: 75849.9 | loss tokens/s: 55448.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.092676E-01 | lm-global-supervised-tokens: 1.240829E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.621618E-02 | lm-token-buffer-mib: 2.331310E+02 | global_load_balancing_loss: 1.000485E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11222.037 | params rms: 0.060764 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:23:30.512011 | finish at 2026-10-17 18:08:22 + [2026-10-12 03:48:36] iteration 223/ 2384 | consumed samples: 7136 | elapsed time per iteration (ms): 224480.5 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 74737.7 | valid tokens/s: 74737.7 | physical tokens/s: 75600.4 | loss tokens/s: 58983.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.806692E-01 | lm-global-supervised-tokens: 1.324058E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.678045E-02 | lm-token-buffer-mib: 2.350955E+02 | global_load_balancing_loss: 9.999196E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11221.971 | params rms: 0.060763 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:45:02.467629 | finish at 2026-10-17 18:33:39 + [2026-10-12 03:52:19] iteration 224/ 2384 | consumed samples: 7168 | elapsed time per iteration (ms): 222756.9 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75316.2 | valid tokens/s: 75316.2 | physical tokens/s: 75975.6 | loss tokens/s: 58289.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.800745E-01 | lm-global-supervised-tokens: 1.298435E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.819039E-02 | lm-token-buffer-mib: 2.345651E+02 | global_load_balancing_loss: 9.997686E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11221.908 | params rms: 0.060763 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:39:14.916000 | finish at 2026-10-17 17:31:34 + [2026-10-12 03:56:02] iteration 225/ 2384 | consumed samples: 7200 | elapsed time per iteration (ms): 222774.0 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75310.4 | valid tokens/s: 75310.4 | physical tokens/s: 75713.8 | loss tokens/s: 54884.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.360832E-01 | lm-global-supervised-tokens: 1.222672E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.739544E-02 | lm-token-buffer-mib: 2.340612E+02 | global_load_balancing_loss: 9.996454E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11221.846 | params rms: 0.060763 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:36:09.031360 | finish at 2026-10-17 17:32:11 + [2026-10-12 03:59:46] iteration 226/ 2384 | consumed samples: 7232 | elapsed time per iteration (ms): 224022.4 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 74890.7 | valid tokens/s: 74890.7 | physical tokens/s: 75782.4 | loss tokens/s: 54508.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.159640E-01 | lm-global-supervised-tokens: 1.221116E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.790946E-02 | lm-token-buffer-mib: 2.355098E+02 | global_load_balancing_loss: 1.000809E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11221.786 | params rms: 0.060762 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:17:20.355968 | finish at 2026-10-17 18:17:06 + [2026-10-12 04:03:28] iteration 227/ 2384 | consumed samples: 7264 | elapsed time per iteration (ms): 222070.4 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.96% | FLOPs estimator: workload-aware | logical tokens/s: 75549.0 | valid tokens/s: 75549.0 | physical tokens/s: 76667.0 | loss tokens/s: 57168.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.143984E-01 | lm-global-supervised-tokens: 1.269537E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.786820E-02 | lm-token-buffer-mib: 2.339737E+02 | global_load_balancing_loss: 1.001479E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11221.721 | params rms: 0.060762 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:03:25.854856 | finish at 2026-10-17 17:06:54 + [2026-10-12 04:08:18] iteration 228/ 2384 | consumed samples: 7296 | elapsed time per iteration (ms): 289874.8 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 57877.4 | valid tokens/s: 57877.4 | physical tokens/s: 59338.0 | loss tokens/s: 42028.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.192205E-01 | lm-global-supervised-tokens: 1.218289E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.270769E-02 | lm-token-buffer-mib: 2.428734E+02 | global_load_balancing_loss: 1.001719E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11221.659 | params rms: 0.060762 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 5:36:10.018867 | finish at 2026-10-19 09:44:28 + [2026-10-12 04:11:58] iteration 229/ 2384 | consumed samples: 7328 | elapsed time per iteration (ms): 220678.1 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.25% | FLOPs estimator: workload-aware | logical tokens/s: 76025.7 | valid tokens/s: 76025.7 | physical tokens/s: 76436.1 | loss tokens/s: 61764.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.274638E-01 | lm-global-supervised-tokens: 1.363002E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.122434E-02 | lm-token-buffer-mib: 2.335746E+02 | global_load_balancing_loss: 1.002284E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11221.598 | params rms: 0.060761 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:06:01.404897 | finish at 2026-10-17 16:18:00 + [2026-10-12 04:16:03] iteration 230/ 2384 | consumed samples: 7360 | elapsed time per iteration (ms): 244757.9 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 68545.9 | valid tokens/s: 68545.9 | physical tokens/s: 69327.2 | loss tokens/s: 50325.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.541623E-01 | lm-global-supervised-tokens: 1.231767E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.445553E-02 | lm-token-buffer-mib: 2.351856E+02 | global_load_balancing_loss: 9.998491E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11221.540 | params rms: 0.060761 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:26:48.561278 | finish at 2026-10-18 06:42:52 + [2026-10-12 04:19:43] iteration 231/ 2384 | consumed samples: 7392 | elapsed time per iteration (ms): 220387.9 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 76125.7 | valid tokens/s: 76125.7 | physical tokens/s: 76847.5 | loss tokens/s: 55515.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.321357E-01 | lm-global-supervised-tokens: 1.223495E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.030552E-02 | lm-token-buffer-mib: 2.356844E+02 | global_load_balancing_loss: 1.000291E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11221.480 | params rms: 0.060761 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:48:15.151512 | finish at 2026-10-17 16:07:59 + [2026-10-12 04:23:26] iteration 232/ 2384 | consumed samples: 7424 | elapsed time per iteration (ms): 222075.9 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 75547.2 | valid tokens/s: 75547.2 | physical tokens/s: 75974.0 | loss tokens/s: 57196.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.148532E-01 | lm-global-supervised-tokens: 1.270199E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.238968E-02 | lm-token-buffer-mib: 2.334944E+02 | global_load_balancing_loss: 1.001493E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11221.421 | params rms: 0.060760 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:45:07.313366 | finish at 2026-10-17 17:08:33 + [2026-10-12 04:27:08] iteration 233/ 2384 | consumed samples: 7456 | elapsed time per iteration (ms): 222018.5 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.26% | FLOPs estimator: workload-aware | logical tokens/s: 75566.6 | valid tokens/s: 75566.6 | physical tokens/s: 75863.1 | loss tokens/s: 53875.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.530282E-01 | lm-global-supervised-tokens: 1.196142E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.544427E-02 | lm-token-buffer-mib: 2.323965E+02 | global_load_balancing_loss: 1.000356E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11221.368 | params rms: 0.060760 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:39:21.797283 | finish at 2026-10-17 17:06:29 + [2026-10-12 04:30:50] iteration 234/ 2384 | consumed samples: 7488 | elapsed time per iteration (ms): 222148.9 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 75522.3 | valid tokens/s: 75522.3 | physical tokens/s: 76060.5 | loss tokens/s: 53150.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.505441E-01 | lm-global-supervised-tokens: 1.180729E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.862340E-02 | lm-token-buffer-mib: 2.344127E+02 | global_load_balancing_loss: 9.996229E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11221.321 | params rms: 0.060760 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:40:20.185816 | finish at 2026-10-17 17:11:10 + [2026-10-12 04:34:31] iteration 235/ 2384 | consumed samples: 7520 | elapsed time per iteration (ms): 221158.8 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.22% | FLOPs estimator: workload-aware | logical tokens/s: 75860.3 | valid tokens/s: 75860.3 | physical tokens/s: 76576.4 | loss tokens/s: 55717.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.217141E-01 | lm-global-supervised-tokens: 1.232249E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.240907E-02 | lm-token-buffer-mib: 2.323340E+02 | global_load_balancing_loss: 9.984648E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11221.275 | params rms: 0.060760 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:01:10.350620 | finish at 2026-10-17 16:35:41 + [2026-10-12 04:38:12] iteration 236/ 2384 | consumed samples: 7552 | elapsed time per iteration (ms): 221307.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 75809.6 | valid tokens/s: 75809.6 | physical tokens/s: 76446.7 | loss tokens/s: 60257.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.592721E-01 | lm-global-supervised-tokens: 1.333545E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.475989E-02 | lm-token-buffer-mib: 2.349065E+02 | global_load_balancing_loss: 1.001926E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11221.223 | params rms: 0.060759 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:02:47.698039 | finish at 2026-10-17 16:41:00 + [2026-10-12 04:41:54] iteration 237/ 2384 | consumed samples: 7584 | elapsed time per iteration (ms): 221814.8 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 75635.9 | valid tokens/s: 75635.9 | physical tokens/s: 76424.4 | loss tokens/s: 57902.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.440415E-01 | lm-global-supervised-tokens: 1.284358E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.774284E-02 | lm-token-buffer-mib: 2.353101E+02 | global_load_balancing_loss: 1.001332E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11221.172 | params rms: 0.060759 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:17:16.429399 | finish at 2026-10-17 16:59:10 + [2026-10-12 04:45:34] iteration 238/ 2384 | consumed samples: 7616 | elapsed time per iteration (ms): 220279.0 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.47% | FLOPs estimator: workload-aware | logical tokens/s: 76163.3 | valid tokens/s: 76163.3 | physical tokens/s: 76522.0 | loss tokens/s: 57121.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.683069E-01 | lm-global-supervised-tokens: 1.258271E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.459551E-02 | lm-token-buffer-mib: 2.336219E+02 | global_load_balancing_loss: 1.001555E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11221.123 | params rms: 0.060759 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:18:38.744839 | finish at 2026-10-17 16:04:13 + [2026-10-12 04:49:16] iteration 239/ 2384 | consumed samples: 7648 | elapsed time per iteration (ms): 221831.0 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.64% | FLOPs estimator: workload-aware | logical tokens/s: 75630.4 | valid tokens/s: 75630.4 | physical tokens/s: 76174.4 | loss tokens/s: 59178.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.969444E-01 | lm-global-supervised-tokens: 1.312754E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.443918E-02 | lm-token-buffer-mib: 2.348155E+02 | global_load_balancing_loss: 1.004305E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11221.077 | params rms: 0.060758 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:10:27.566789 | finish at 2026-10-17 16:59:44 + [2026-10-12 04:53:20] iteration 240/ 2384 | consumed samples: 7680 | elapsed time per iteration (ms): 244358.8 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.11% | FLOPs estimator: workload-aware | logical tokens/s: 68658.1 | valid tokens/s: 68658.1 | physical tokens/s: 69317.6 | loss tokens/s: 52898.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.785318E-01 | lm-global-supervised-tokens: 1.292631E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.923766E-02 | lm-token-buffer-mib: 2.338378E+02 | global_load_balancing_loss: 1.000487E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11221.031 | params rms: 0.060758 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:31:45.180672 | finish at 2026-10-18 06:25:06 +times across ranks (ms): + save-checkpoint: + rank 0: 24139.97 + rank 1: 24139.96 + rank 2: 24139.95 + rank 3: 24139.99 + rank 4: 24139.97 + rank 5: 24139.96 + rank 6: 24139.96 + rank 7: 24139.98 + rank 8: 24139.62 + rank 9: 24139.63 + rank 10: 24139.62 + rank 11: 24139.62 + rank 12: 24139.61 + rank 13: 24139.63 + rank 14: 24139.64 + rank 15: 24139.64 + rank 16: 24139.68 + rank 17: 24139.69 + rank 18: 24139.69 + rank 19: 24139.69 + rank 20: 24139.68 + rank 21: 24139.69 + rank 22: 24139.65 + rank 23: 24139.67 + rank 24: 24139.32 + rank 25: 24139.31 + rank 26: 24139.31 + rank 27: 24139.34 + rank 28: 24139.31 + rank 29: 24139.35 + rank 30: 24139.39 + rank 31: 24139.32 + rank 32: 24139.36 + rank 33: 24139.32 + rank 34: 24139.36 + rank 35: 24139.41 + rank 36: 24139.32 + rank 37: 24139.36 + rank 38: 24139.31 + rank 39: 24139.30 + rank 40: 24139.51 + rank 41: 24139.45 + rank 42: 24139.43 + rank 43: 24139.44 + rank 44: 24139.43 + rank 45: 24139.43 + rank 46: 24139.47 + rank 47: 24139.44 + rank 48: 24139.46 + rank 49: 24139.38 + rank 50: 24139.40 + rank 51: 24139.37 + rank 52: 24139.36 + rank 53: 24139.41 + rank 54: 24139.35 + rank 55: 24139.35 + rank 56: 24139.65 + rank 57: 24139.66 + rank 58: 24139.65 + rank 59: 24139.63 + rank 60: 24139.63 + rank 61: 24139.63 + rank 62: 24139.62 + rank 63: 24139.62 + [2026-10-12 04:57:33] iteration 241/ 2384 | consumed samples: 7712 | elapsed time per iteration (ms): 228646.2 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.76% | FLOPs estimator: workload-aware | logical tokens/s: 73376.2 | valid tokens/s: 73376.2 | physical tokens/s: 74442.5 | loss tokens/s: 57246.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.000616E-01 | lm-global-supervised-tokens: 1.308920E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.116298E-01 | lm-token-buffer-mib: 2.343742E+02 | global_load_balancing_loss: 9.996887E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11220.982 | params rms: 0.060758 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:06:28.863188 | finish at 2026-10-17 21:04:02 + [2026-10-12 05:01:17] iteration 242/ 2384 | consumed samples: 7744 | elapsed time per iteration (ms): 224119.5 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.20% | FLOPs estimator: workload-aware | logical tokens/s: 74858.2 | valid tokens/s: 74858.2 | physical tokens/s: 75588.0 | loss tokens/s: 54415.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.162785E-01 | lm-global-supervised-tokens: 1.219564E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.340370E-02 | lm-token-buffer-mib: 2.331894E+02 | global_load_balancing_loss: 9.998192E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11220.935 | params rms: 0.060758 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:21:03.985175 | finish at 2026-10-17 18:22:21 + [2026-10-12 05:04:58] iteration 243/ 2384 | consumed samples: 7776 | elapsed time per iteration (ms): 220242.6 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 76176.0 | valid tokens/s: 76176.0 | physical tokens/s: 76606.4 | loss tokens/s: 57870.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.555362E-01 | lm-global-supervised-tokens: 1.274557E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.583848E-02 | lm-token-buffer-mib: 2.337889E+02 | global_load_balancing_loss: 9.994828E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11220.888 | params rms: 0.060757 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:58:59.342206 | finish at 2026-10-17 16:03:57 + [2026-10-12 05:08:39] iteration 244/ 2384 | consumed samples: 7808 | elapsed time per iteration (ms): 221546.8 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 75727.5 | valid tokens/s: 75727.5 | physical tokens/s: 77056.2 | loss tokens/s: 58477.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.007762E-01 | lm-global-supervised-tokens: 1.295554E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.615956E-02 | lm-token-buffer-mib: 2.348928E+02 | global_load_balancing_loss: 9.991294E-01 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11220.842 | params rms: 0.060757 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:41:50.215049 | finish at 2026-10-17 16:50:29 + [2026-10-12 05:12:22] iteration 245/ 2384 | consumed samples: 7840 | elapsed time per iteration (ms): 222339.0 | throughput per GPU (TFLOP/s/GPU): 25.4 | MFU 8.15% | FLOPs estimator: workload-aware | logical tokens/s: 75457.7 | valid tokens/s: 75457.7 | physical tokens/s: 76147.2 | loss tokens/s: 56860.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.302092E-01 | lm-global-supervised-tokens: 1.264219E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.602476E-02 | lm-token-buffer-mib: 2.335096E+02 | global_load_balancing_loss: 9.998875E-01 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11220.792 | params rms: 0.060757 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:06:23.080173 | finish at 2026-10-17 17:18:45 + [2026-10-12 05:16:01] iteration 246/ 2384 | consumed samples: 7872 | elapsed time per iteration (ms): 219974.8 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 76268.7 | valid tokens/s: 76268.7 | physical tokens/s: 77475.7 | loss tokens/s: 60623.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.789813E-01 | lm-global-supervised-tokens: 1.333554E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.536498E-02 | lm-token-buffer-mib: 2.346447E+02 | global_load_balancing_loss: 9.996771E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11220.739 | params rms: 0.060757 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:38:26.037508 | finish at 2026-10-17 15:54:28 + [2026-10-12 05:19:43] iteration 247/ 2384 | consumed samples: 7904 | elapsed time per iteration (ms): 221542.9 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75728.8 | valid tokens/s: 75728.8 | physical tokens/s: 76660.3 | loss tokens/s: 60036.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.811696E-01 | lm-global-supervised-tokens: 1.330064E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.285958E-02 | lm-token-buffer-mib: 2.346760E+02 | global_load_balancing_loss: 1.001471E+00 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11220.690 | params rms: 0.060756 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:30:37.270212 | finish at 2026-10-17 16:50:20 + [2026-10-12 05:23:25] iteration 248/ 2384 | consumed samples: 7936 | elapsed time per iteration (ms): 222191.4 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75507.9 | valid tokens/s: 75507.9 | physical tokens/s: 75918.4 | loss tokens/s: 58361.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.011032E-01 | lm-global-supervised-tokens: 1.296731E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.983270E-02 | lm-token-buffer-mib: 2.324781E+02 | global_load_balancing_loss: 1.000843E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11220.638 | params rms: 0.060756 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:50:00.792315 | finish at 2026-10-17 17:13:26 + [2026-10-12 05:27:09] iteration 249/ 2384 | consumed samples: 7968 | elapsed time per iteration (ms): 223478.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75073.1 | valid tokens/s: 75073.1 | physical tokens/s: 76116.2 | loss tokens/s: 58240.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.139420E-01 | lm-global-supervised-tokens: 1.301535E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.057443E-02 | lm-token-buffer-mib: 2.356631E+02 | global_load_balancing_loss: 1.001475E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11220.587 | params rms: 0.060756 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:32:05.654043 | finish at 2026-10-17 17:59:14 + [2026-10-12 05:31:16] iteration 250/ 2384 | consumed samples: 8000 | elapsed time per iteration (ms): 247706.8 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 6.01% | FLOPs estimator: workload-aware | logical tokens/s: 67730.0 | valid tokens/s: 67730.0 | physical tokens/s: 68866.7 | loss tokens/s: 53434.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.286543E-01 | lm-global-supervised-tokens: 1.323601E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.228350E-02 | lm-token-buffer-mib: 2.377425E+02 | global_load_balancing_loss: 1.001037E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11220.539 | params rms: 0.060756 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:50:06.296411 | finish at 2026-10-18 08:21:23 + [2026-10-12 05:35:00] iteration 251/ 2384 | consumed samples: 8032 | elapsed time per iteration (ms): 223560.2 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 75045.5 | valid tokens/s: 75045.5 | physical tokens/s: 75921.4 | loss tokens/s: 60466.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.516382E-01 | lm-global-supervised-tokens: 1.351793E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.738134E-02 | lm-token-buffer-mib: 2.345603E+02 | global_load_balancing_loss: 1.000954E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11220.492 | params rms: 0.060755 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:27:33.847557 | finish at 2026-10-17 18:02:34 + [2026-10-12 05:38:45] iteration 252/ 2384 | consumed samples: 8064 | elapsed time per iteration (ms): 225536.5 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 74387.8 | valid tokens/s: 74387.8 | physical tokens/s: 75418.7 | loss tokens/s: 53401.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.995020E-01 | lm-global-supervised-tokens: 1.204400E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.145633E-02 | lm-token-buffer-mib: 2.352197E+02 | global_load_balancing_loss: 1.000214E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11220.441 | params rms: 0.060755 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:34:03.859632 | finish at 2026-10-17 19:12:49 + [2026-10-12 05:42:27] iteration 253/ 2384 | consumed samples: 8096 | elapsed time per iteration (ms): 221296.3 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 75813.1 | valid tokens/s: 75813.1 | physical tokens/s: 76344.9 | loss tokens/s: 58226.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.369431E-01 | lm-global-supervised-tokens: 1.288520E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.978287E-02 | lm-token-buffer-mib: 2.334887E+02 | global_load_balancing_loss: 1.000939E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11220.391 | params rms: 0.060755 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:59:42.451741 | finish at 2026-10-17 16:42:09 + [2026-10-12 05:46:15] iteration 254/ 2384 | consumed samples: 8128 | elapsed time per iteration (ms): 228400.2 | throughput per GPU (TFLOP/s/GPU): 25.5 | MFU 8.18% | FLOPs estimator: workload-aware | logical tokens/s: 73455.1 | valid tokens/s: 73455.1 | physical tokens/s: 74387.6 | loss tokens/s: 55453.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.441933E-01 | lm-global-supervised-tokens: 1.266559E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.265207E-02 | lm-token-buffer-mib: 2.351319E+02 | global_load_balancing_loss: 9.999667E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11220.341 | params rms: 0.060754 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:08:12.497878 | finish at 2026-10-17 20:54:28 + [2026-10-12 05:49:58] iteration 255/ 2384 | consumed samples: 8160 | elapsed time per iteration (ms): 222335.5 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 75458.8 | valid tokens/s: 75458.8 | physical tokens/s: 76614.7 | loss tokens/s: 57783.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.789344E-01 | lm-global-supervised-tokens: 1.284726E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.521435E-02 | lm-token-buffer-mib: 2.360622E+02 | global_load_balancing_loss: 1.000430E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11220.282 | params rms: 0.060754 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:29:12.345999 | finish at 2026-10-17 17:19:10 + [2026-10-12 05:53:40] iteration 256/ 2384 | consumed samples: 8192 | elapsed time per iteration (ms): 222795.5 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 75303.2 | valid tokens/s: 75303.2 | physical tokens/s: 75663.1 | loss tokens/s: 57312.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.671052E-01 | lm-global-supervised-tokens: 1.276892E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.774108E-02 | lm-token-buffer-mib: 2.329684E+02 | global_load_balancing_loss: 9.994578E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11220.224 | params rms: 0.060754 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:41:48.798210 | finish at 2026-10-17 17:35:29 + [2026-10-12 05:57:24] iteration 257/ 2384 | consumed samples: 8224 | elapsed time per iteration (ms): 223290.9 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75136.1 | valid tokens/s: 75136.1 | physical tokens/s: 75889.1 | loss tokens/s: 55383.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.836191E-01 | lm-global-supervised-tokens: 1.236653E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.950451E-02 | lm-token-buffer-mib: 2.346905E+02 | global_load_balancing_loss: 9.997501E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11220.168 | params rms: 0.060754 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:55:39.738705 | finish at 2026-10-17 17:53:03 + [2026-10-12 06:01:05] iteration 258/ 2384 | consumed samples: 8256 | elapsed time per iteration (ms): 221165.3 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 75858.2 | valid tokens/s: 75858.2 | physical tokens/s: 76582.6 | loss tokens/s: 57721.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.430092E-01 | lm-global-supervised-tokens: 1.276597E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.614901E-02 | lm-token-buffer-mib: 2.353087E+02 | global_load_balancing_loss: 1.000750E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11220.113 | params rms: 0.060753 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:36:37.354074 | finish at 2026-10-17 16:37:42 + [2026-10-12 06:04:48] iteration 259/ 2384 | consumed samples: 8288 | elapsed time per iteration (ms): 223161.4 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.29% | FLOPs estimator: workload-aware | logical tokens/s: 75179.7 | valid tokens/s: 75179.6 | physical tokens/s: 75653.2 | loss tokens/s: 53428.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.592942E-01 | lm-global-supervised-tokens: 1.192321E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.683256E-02 | lm-token-buffer-mib: 2.344889E+02 | global_load_balancing_loss: 1.002049E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11220.057 | params rms: 0.060753 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:43:37.888534 | finish at 2026-10-17 17:48:26 + [2026-10-12 06:08:48] iteration 260/ 2384 | consumed samples: 8320 | elapsed time per iteration (ms): 240451.6 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.09% | FLOPs estimator: workload-aware | logical tokens/s: 69773.6 | valid tokens/s: 69773.6 | physical tokens/s: 70691.5 | loss tokens/s: 55138.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.519764E-01 | lm-global-supervised-tokens: 1.325810E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.112807E-02 | lm-token-buffer-mib: 2.338159E+02 | global_load_balancing_loss: 9.992888E-01 | loss scale: 1.0 | grad norm: 0.223 | params norm: 11219.999 | params rms: 0.060753 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:51:59.097279 | finish at 2026-10-18 04:00:47 + [2026-10-12 06:12:27] iteration 261/ 2384 | consumed samples: 8352 | elapsed time per iteration (ms): 218559.9 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.14% | FLOPs estimator: workload-aware | logical tokens/s: 76762.6 | valid tokens/s: 76762.6 | physical tokens/s: 77216.4 | loss tokens/s: 57024.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.175370E-01 | lm-global-supervised-tokens: 1.246316E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.586534E-02 | lm-token-buffer-mib: 2.328784E+02 | global_load_balancing_loss: 1.000715E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11219.941 | params rms: 0.060752 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:53:22.640464 | finish at 2026-10-17 15:05:50 + [2026-10-12 06:16:11] iteration 262/ 2384 | consumed samples: 8384 | elapsed time per iteration (ms): 223573.8 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75041.0 | valid tokens/s: 75041.0 | physical tokens/s: 75765.6 | loss tokens/s: 53631.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.977711E-01 | lm-global-supervised-tokens: 1.199066E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.618423E-02 | lm-token-buffer-mib: 2.350622E+02 | global_load_balancing_loss: 1.001930E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11219.883 | params rms: 0.060752 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:47:03.624022 | finish at 2026-10-17 18:03:14 + [2026-10-12 06:19:53] iteration 263/ 2384 | consumed samples: 8416 | elapsed time per iteration (ms): 222743.5 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 75320.8 | valid tokens/s: 75320.8 | physical tokens/s: 76020.7 | loss tokens/s: 59067.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.802726E-01 | lm-global-supervised-tokens: 1.315687E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.283065E-02 | lm-token-buffer-mib: 2.330451E+02 | global_load_balancing_loss: 1.000290E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11219.827 | params rms: 0.060752 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:13:58.927078 | finish at 2026-10-17 17:33:52 + [2026-10-12 06:23:37] iteration 264/ 2384 | consumed samples: 8448 | elapsed time per iteration (ms): 223971.3 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 74907.8 | valid tokens/s: 74907.8 | physical tokens/s: 75796.6 | loss tokens/s: 58535.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.806686E-01 | lm-global-supervised-tokens: 1.311018E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.634159E-02 | lm-token-buffer-mib: 2.347899E+02 | global_load_balancing_loss: 1.000091E+00 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11219.772 | params rms: 0.060751 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:53:39.161825 | finish at 2026-10-17 18:17:16 + [2026-10-12 06:27:20] iteration 265/ 2384 | consumed samples: 8480 | elapsed time per iteration (ms): 223091.7 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75203.1 | valid tokens/s: 75203.1 | physical tokens/s: 75821.2 | loss tokens/s: 56842.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.602536E-01 | lm-global-supervised-tokens: 1.268117E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.944335E-02 | lm-token-buffer-mib: 2.325260E+02 | global_load_balancing_loss: 1.000648E+00 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11219.717 | params rms: 0.060751 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:18:51.277099 | finish at 2026-10-17 17:46:12 + [2026-10-12 06:31:03] iteration 266/ 2384 | consumed samples: 8512 | elapsed time per iteration (ms): 223140.2 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 75186.8 | valid tokens/s: 75186.8 | physical tokens/s: 75654.4 | loss tokens/s: 58490.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.475256E-01 | lm-global-supervised-tokens: 1.305168E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.777215E-02 | lm-token-buffer-mib: 2.336024E+02 | global_load_balancing_loss: 1.000295E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11219.667 | params rms: 0.060751 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:16:51.014083 | finish at 2026-10-17 17:47:54 + [2026-10-12 06:34:49] iteration 267/ 2384 | consumed samples: 8544 | elapsed time per iteration (ms): 225440.9 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 74419.4 | valid tokens/s: 74419.4 | physical tokens/s: 75565.1 | loss tokens/s: 57100.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.871708E-01 | lm-global-supervised-tokens: 1.287277E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.473239E-02 | lm-token-buffer-mib: 2.380558E+02 | global_load_balancing_loss: 1.001686E+00 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11219.617 | params rms: 0.060751 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:34:18.430911 | finish at 2026-10-17 19:09:07 + [2026-10-12 06:38:30] iteration 268/ 2384 | consumed samples: 8576 | elapsed time per iteration (ms): 221209.8 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75842.9 | valid tokens/s: 75842.9 | physical tokens/s: 76247.5 | loss tokens/s: 56420.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.733246E-01 | lm-global-supervised-tokens: 1.248081E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.305457E-02 | lm-token-buffer-mib: 2.335947E+02 | global_load_balancing_loss: 9.998124E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11219.566 | params rms: 0.060750 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:01:19.899982 | finish at 2026-10-17 16:39:50 + [2026-10-12 06:42:11] iteration 269/ 2384 | consumed samples: 8608 | elapsed time per iteration (ms): 220440.0 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 76107.8 | valid tokens/s: 76107.7 | physical tokens/s: 77084.8 | loss tokens/s: 55729.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.760448E-01 | lm-global-supervised-tokens: 1.228501E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.110591E-02 | lm-token-buffer-mib: 2.337684E+02 | global_load_balancing_loss: 9.997844E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11219.515 | params rms: 0.060750 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:30:30.549695 | finish at 2026-10-17 16:12:41 + [2026-10-12 06:47:07] iteration 270/ 2384 | consumed samples: 8640 | elapsed time per iteration (ms): 296676.9 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.13% | FLOPs estimator: workload-aware | logical tokens/s: 56550.4 | valid tokens/s: 56550.4 | physical tokens/s: 57608.4 | loss tokens/s: 38999.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.936493E-01 | lm-global-supervised-tokens: 1.157017E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.578068E-02 | lm-token-buffer-mib: 2.340768E+02 | global_load_balancing_loss: 1.000514E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11219.464 | params rms: 0.060750 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 6:12:54.962378 | finish at 2026-10-19 13:00:02 + [2026-10-12 06:50:51] iteration 271/ 2384 | consumed samples: 8672 | elapsed time per iteration (ms): 223886.2 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 74936.3 | valid tokens/s: 74936.3 | physical tokens/s: 76105.0 | loss tokens/s: 58966.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.757211E-01 | lm-global-supervised-tokens: 1.320188E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.286948E-02 | lm-token-buffer-mib: 2.357536E+02 | global_load_balancing_loss: 1.002124E+00 | loss scale: 1.0 | grad norm: 0.194 | params norm: 11219.413 | params rms: 0.060749 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:24:31.537474 | finish at 2026-10-17 18:15:23 + [2026-10-12 06:54:37] iteration 272/ 2384 | consumed samples: 8704 | elapsed time per iteration (ms): 226130.3 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 74192.4 | valid tokens/s: 74192.4 | physical tokens/s: 74813.9 | loss tokens/s: 55825.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.720490E-01 | lm-global-supervised-tokens: 1.262391E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.575923E-02 | lm-token-buffer-mib: 2.328498E+02 | global_load_balancing_loss: 1.002040E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11219.362 | params rms: 0.060749 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:39:47.190674 | finish at 2026-10-17 19:34:24 + [2026-10-12 06:58:16] iteration 273/ 2384 | consumed samples: 8736 | elapsed time per iteration (ms): 218933.3 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 76631.5 | valid tokens/s: 76631.4 | physical tokens/s: 77017.2 | loss tokens/s: 54338.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.216733E-01 | lm-global-supervised-tokens: 1.189642E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.343298E-02 | lm-token-buffer-mib: 2.330509E+02 | global_load_balancing_loss: 1.002254E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11219.311 | params rms: 0.060749 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:22:48.193319 | finish at 2026-10-17 15:21:04 + [2026-10-12 07:01:59] iteration 274/ 2384 | consumed samples: 8768 | elapsed time per iteration (ms): 223166.0 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.05% | FLOPs estimator: workload-aware | logical tokens/s: 75178.1 | valid tokens/s: 75178.1 | physical tokens/s: 76687.5 | loss tokens/s: 56379.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.890970E-01 | lm-global-supervised-tokens: 1.258190E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.315738E-02 | lm-token-buffer-mib: 2.387961E+02 | global_load_balancing_loss: 1.001203E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11219.262 | params rms: 0.060749 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:48:00.219522 | finish at 2026-10-17 17:50:00 + [2026-10-12 07:05:39] iteration 275/ 2384 | consumed samples: 8800 | elapsed time per iteration (ms): 219815.8 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.29% | FLOPs estimator: workload-aware | logical tokens/s: 76323.7 | valid tokens/s: 76323.7 | physical tokens/s: 76689.1 | loss tokens/s: 57353.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.404466E-01 | lm-global-supervised-tokens: 1.260722E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.305167E-02 | lm-token-buffer-mib: 2.338399E+02 | global_load_balancing_loss: 9.993363E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11219.210 | params rms: 0.060748 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:46:31.588974 | finish at 2026-10-17 15:52:11 + [2026-10-12 07:09:24] iteration 276/ 2384 | consumed samples: 8832 | elapsed time per iteration (ms): 224411.9 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 74760.7 | valid tokens/s: 74760.7 | physical tokens/s: 76027.3 | loss tokens/s: 60235.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.925493E-01 | lm-global-supervised-tokens: 1.351744E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.551375E-02 | lm-token-buffer-mib: 2.347618E+02 | global_load_balancing_loss: 9.985445E-01 | loss scale: 1.0 | grad norm: 0.477 | params norm: 11219.154 | params rms: 0.060748 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:24:20.358167 | finish at 2026-10-17 18:33:44 + [2026-10-12 07:14:05] iteration 277/ 2384 | consumed samples: 8864 | elapsed time per iteration (ms): 281423.3 | throughput per GPU (TFLOP/s/GPU): 17.4 | MFU 5.57% | FLOPs estimator: workload-aware | logical tokens/s: 59615.5 | valid tokens/s: 59615.5 | physical tokens/s: 60816.6 | loss tokens/s: 43835.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.522072E-01 | lm-global-supervised-tokens: 1.233621E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.043626E-02 | lm-token-buffer-mib: 2.407329E+02 | global_load_balancing_loss: 9.999393E-01 | loss scale: 1.0 | grad norm: 0.381 | params norm: 11219.097 | params rms: 0.060748 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:42:38.868544 | finish at 2026-10-19 03:56:44 + [2026-10-12 07:17:49] iteration 278/ 2384 | consumed samples: 8896 | elapsed time per iteration (ms): 223513.5 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 75061.3 | valid tokens/s: 75061.3 | physical tokens/s: 75931.5 | loss tokens/s: 57164.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.923021E-01 | lm-global-supervised-tokens: 1.277714E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.724997E-02 | lm-token-buffer-mib: 2.356483E+02 | global_load_balancing_loss: 1.000625E+00 | loss scale: 1.0 | grad norm: 0.324 | params norm: 11219.027 | params rms: 0.060747 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:45:19.490197 | finish at 2026-10-17 18:03:08 + [2026-10-12 07:21:30] iteration 279/ 2384 | consumed samples: 8928 | elapsed time per iteration (ms): 221206.5 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 75844.0 | valid tokens/s: 75844.0 | physical tokens/s: 76416.7 | loss tokens/s: 60473.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.451239E-01 | lm-global-supervised-tokens: 1.337701E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.095069E-02 | lm-token-buffer-mib: 2.336172E+02 | global_load_balancing_loss: 1.001311E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11218.965 | params rms: 0.060747 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:20:39.734305 | finish at 2026-10-17 16:42:09 + [2026-10-12 07:25:31] iteration 280/ 2384 | consumed samples: 8960 | elapsed time per iteration (ms): 241219.6 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 69551.6 | valid tokens/s: 69551.6 | physical tokens/s: 69942.7 | loss tokens/s: 49075.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.759549E-01 | lm-global-supervised-tokens: 1.183800E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.499307E-02 | lm-token-buffer-mib: 2.322530E+02 | global_load_balancing_loss: 1.001139E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11218.912 | params rms: 0.060747 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:58:45.948528 | finish at 2026-10-18 04:24:17 +times across ranks (ms): + save-checkpoint: + rank 0: 24560.59 + rank 1: 24560.60 + rank 2: 24560.59 + rank 3: 24560.56 + rank 4: 24560.55 + rank 5: 24560.59 + rank 6: 24560.59 + rank 7: 24560.59 + rank 8: 24561.95 + rank 9: 24561.99 + rank 10: 24561.97 + rank 11: 24561.95 + rank 12: 24561.96 + rank 13: 24561.95 + rank 14: 24561.96 + rank 15: 24561.95 + rank 16: 24562.21 + rank 17: 24562.19 + rank 18: 24562.19 + rank 19: 24562.19 + rank 20: 24562.17 + rank 21: 24562.18 + rank 22: 24562.18 + rank 23: 24562.18 + rank 24: 24562.81 + rank 25: 24562.85 + rank 26: 24562.83 + rank 27: 24562.82 + rank 28: 24562.86 + rank 29: 24562.80 + rank 30: 24562.80 + rank 31: 24562.81 + rank 32: 24561.64 + rank 33: 24561.65 + rank 34: 24561.66 + rank 35: 24561.77 + rank 36: 24561.74 + rank 37: 24561.67 + rank 38: 24561.60 + rank 39: 24561.52 + rank 40: 24560.64 + rank 41: 24560.62 + rank 42: 24560.62 + rank 43: 24560.64 + rank 44: 24560.61 + rank 45: 24560.48 + rank 46: 24560.59 + rank 47: 24560.60 + rank 48: 24561.62 + rank 49: 24561.50 + rank 50: 24561.45 + rank 51: 24561.53 + rank 52: 24561.42 + rank 53: 24561.42 + rank 54: 24561.42 + rank 55: 24561.49 + rank 56: 24562.40 + rank 57: 24562.44 + rank 58: 24562.44 + rank 59: 24562.42 + rank 60: 24562.43 + rank 61: 24562.41 + rank 62: 24562.39 + rank 63: 24562.38 + [2026-10-12 07:30:01] iteration 281/ 2384 | consumed samples: 8992 | elapsed time per iteration (ms): 245100.8 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 68450.1 | valid tokens/s: 68450.1 | physical tokens/s: 69251.7 | loss tokens/s: 51924.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.583592E-01 | lm-global-supervised-tokens: 1.272670E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 9.029455E-01 | lm-token-buffer-mib: 2.354636E+02 | global_load_balancing_loss: 1.000887E+00 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11218.865 | params rms: 0.060747 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:10:46.892729 | finish at 2026-10-18 06:40:48 + [2026-10-12 07:33:49] iteration 282/ 2384 | consumed samples: 9024 | elapsed time per iteration (ms): 227758.1 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.65% | FLOPs estimator: workload-aware | logical tokens/s: 73662.1 | valid tokens/s: 73662.1 | physical tokens/s: 74712.8 | loss tokens/s: 59792.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.141860E-01 | lm-global-supervised-tokens: 1.361825E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.017251E-02 | lm-token-buffer-mib: 2.332372E+02 | global_load_balancing_loss: 1.000993E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11218.824 | params rms: 0.060746 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:59:07.607456 | finish at 2026-10-17 20:32:56 + [2026-10-12 07:37:34] iteration 283/ 2384 | consumed samples: 9056 | elapsed time per iteration (ms): 225521.4 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 74392.9 | valid tokens/s: 74392.9 | physical tokens/s: 74871.5 | loss tokens/s: 55196.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.121827E-01 | lm-global-supervised-tokens: 1.244789E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.336093E-02 | lm-token-buffer-mib: 2.339239E+02 | global_load_balancing_loss: 9.998235E-01 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11218.784 | params rms: 0.060746 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:37:00.414762 | finish at 2026-10-17 19:14:35 + [2026-10-12 07:41:16] iteration 284/ 2384 | consumed samples: 9088 | elapsed time per iteration (ms): 221345.3 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.81% | FLOPs estimator: workload-aware | logical tokens/s: 75796.3 | valid tokens/s: 75796.3 | physical tokens/s: 76895.0 | loss tokens/s: 58828.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.397047E-01 | lm-global-supervised-tokens: 1.302149E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.749711E-02 | lm-token-buffer-mib: 2.330570E+02 | global_load_balancing_loss: 1.001275E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11218.745 | params rms: 0.060746 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:07:05.207019 | finish at 2026-10-17 16:48:21 + [2026-10-12 07:44:59] iteration 285/ 2384 | consumed samples: 9120 | elapsed time per iteration (ms): 223236.9 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.41% | FLOPs estimator: workload-aware | logical tokens/s: 75154.0 | valid tokens/s: 75154.0 | physical tokens/s: 75790.8 | loss tokens/s: 59472.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.697906E-01 | lm-global-supervised-tokens: 1.327641E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.774359E-02 | lm-token-buffer-mib: 2.330679E+02 | global_load_balancing_loss: 1.002509E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11218.707 | params rms: 0.060746 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:09:34.341369 | finish at 2026-10-17 17:54:33 + [2026-10-12 07:48:40] iteration 286/ 2384 | consumed samples: 9152 | elapsed time per iteration (ms): 221393.5 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 75780.0 | valid tokens/s: 75779.9 | physical tokens/s: 76429.6 | loss tokens/s: 57795.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.225835E-01 | lm-global-supervised-tokens: 1.279558E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.488539E-02 | lm-token-buffer-mib: 2.331330E+02 | global_load_balancing_loss: 1.000137E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11218.666 | params rms: 0.060745 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:01:23.498162 | finish at 2026-10-17 16:50:04 + [2026-10-12 07:52:25] iteration 287/ 2384 | consumed samples: 9184 | elapsed time per iteration (ms): 225090.4 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.26% | FLOPs estimator: workload-aware | logical tokens/s: 74535.3 | valid tokens/s: 74535.3 | physical tokens/s: 74953.4 | loss tokens/s: 53127.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.232671E-01 | lm-global-supervised-tokens: 1.195847E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.149678E-02 | lm-token-buffer-mib: 2.333488E+02 | global_load_balancing_loss: 1.001787E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11218.625 | params rms: 0.060745 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:06:54.588758 | finish at 2026-10-17 18:59:20 + [2026-10-12 07:56:05] iteration 288/ 2384 | consumed samples: 9216 | elapsed time per iteration (ms): 219547.9 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.20% | FLOPs estimator: workload-aware | logical tokens/s: 76416.9 | valid tokens/s: 76416.9 | physical tokens/s: 77838.2 | loss tokens/s: 56722.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.544781E-01 | lm-global-supervised-tokens: 1.245338E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.703277E-02 | lm-token-buffer-mib: 2.335863E+02 | global_load_balancing_loss: 9.990307E-01 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11218.583 | params rms: 0.060745 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:49:32.380829 | finish at 2026-10-17 15:45:37 + [2026-10-12 07:59:48] iteration 289/ 2384 | consumed samples: 9248 | elapsed time per iteration (ms): 223497.7 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 75066.6 | valid tokens/s: 75066.6 | physical tokens/s: 75886.5 | loss tokens/s: 54386.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.340811E-01 | lm-global-supervised-tokens: 1.215514E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.972701E-02 | lm-token-buffer-mib: 2.344211E+02 | global_load_balancing_loss: 9.985932E-01 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11218.540 | params rms: 0.060745 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:03:47.659972 | finish at 2026-10-17 18:03:36 + [2026-10-12 08:03:50] iteration 290/ 2384 | consumed samples: 9280 | elapsed time per iteration (ms): 241464.9 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 69480.8 | valid tokens/s: 69480.8 | physical tokens/s: 70131.7 | loss tokens/s: 55293.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.279600E-01 | lm-global-supervised-tokens: 1.335131E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.786601E-02 | lm-token-buffer-mib: 2.342965E+02 | global_load_balancing_loss: 9.980164E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11218.494 | params rms: 0.060744 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:27:07.556551 | finish at 2026-10-18 04:30:57 + [2026-10-12 08:07:32] iteration 291/ 2384 | consumed samples: 9312 | elapsed time per iteration (ms): 222256.5 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75485.6 | valid tokens/s: 75485.6 | physical tokens/s: 76035.0 | loss tokens/s: 59988.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.943010E-01 | lm-global-supervised-tokens: 1.333285E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.576505E-02 | lm-token-buffer-mib: 2.330204E+02 | global_load_balancing_loss: 9.988183E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11218.447 | params rms: 0.060744 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:13:02.881958 | finish at 2026-10-17 17:20:35 + [2026-10-12 08:12:07] iteration 292/ 2384 | consumed samples: 9344 | elapsed time per iteration (ms): 275092.3 | throughput per GPU (TFLOP/s/GPU): 16.6 | MFU 5.31% | FLOPs estimator: workload-aware | logical tokens/s: 60987.4 | valid tokens/s: 60987.4 | physical tokens/s: 62109.6 | loss tokens/s: 44949.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.307436E-01 | lm-global-supervised-tokens: 1.236525E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.195083E-02 | lm-token-buffer-mib: 2.382224E+02 | global_load_balancing_loss: 1.001233E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11218.400 | params rms: 0.060744 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:51:33.185240 | finish at 2026-10-19 00:03:40 + [2026-10-12 08:15:51] iteration 293/ 2384 | consumed samples: 9376 | elapsed time per iteration (ms): 223545.6 | throughput per GPU (TFLOP/s/GPU): 16.9 | MFU 5.43% | FLOPs estimator: workload-aware | logical tokens/s: 75050.4 | valid tokens/s: 75050.4 | physical tokens/s: 76819.0 | loss tokens/s: 63058.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.219398E-01 | lm-global-supervised-tokens: 1.409635E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.990740E-02 | lm-token-buffer-mib: 2.358413E+02 | global_load_balancing_loss: 1.001259E+00 | loss scale: 1.0 | grad norm: 0.244 | params norm: 11218.354 | params rms: 0.060744 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:50:33.935217 | finish at 2026-10-17 18:06:25 + [2026-10-12 08:19:34] iteration 294/ 2384 | consumed samples: 9408 | elapsed time per iteration (ms): 223249.6 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75149.9 | valid tokens/s: 75149.9 | physical tokens/s: 75718.5 | loss tokens/s: 54104.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.594301E-01 | lm-global-supervised-tokens: 1.207879E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.315774E-02 | lm-token-buffer-mib: 2.337018E+02 | global_load_balancing_loss: 1.003997E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11218.308 | params rms: 0.060743 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:36:31.650906 | finish at 2026-10-17 17:56:06 + [2026-10-12 08:23:17] iteration 295/ 2384 | consumed samples: 9440 | elapsed time per iteration (ms): 223128.7 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.21% | FLOPs estimator: workload-aware | logical tokens/s: 75190.6 | valid tokens/s: 75190.6 | physical tokens/s: 75508.6 | loss tokens/s: 56760.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.276405E-01 | lm-global-supervised-tokens: 1.266486E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.746167E-02 | lm-token-buffer-mib: 2.334030E+02 | global_load_balancing_loss: 1.003305E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11218.258 | params rms: 0.060743 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:28:35.868283 | finish at 2026-10-17 17:51:53 + [2026-10-12 08:27:03] iteration 296/ 2384 | consumed samples: 9472 | elapsed time per iteration (ms): 226219.5 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 74163.2 | valid tokens/s: 74163.2 | physical tokens/s: 75157.9 | loss tokens/s: 57961.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.806692E-01 | lm-global-supervised-tokens: 1.311210E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.129389E-02 | lm-token-buffer-mib: 2.364314E+02 | global_load_balancing_loss: 1.001496E+00 | loss scale: 1.0 | grad norm: 0.250 | params norm: 11218.208 | params rms: 0.060743 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:12:26.290249 | finish at 2026-10-17 19:39:30 + [2026-10-12 08:30:51] iteration 297/ 2384 | consumed samples: 9504 | elapsed time per iteration (ms): 227235.5 | throughput per GPU (TFLOP/s/GPU): 18.3 | MFU 5.86% | FLOPs estimator: workload-aware | logical tokens/s: 73831.7 | valid tokens/s: 73831.7 | physical tokens/s: 74995.3 | loss tokens/s: 55389.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.560395E-01 | lm-global-supervised-tokens: 1.258644E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.350246E-02 | lm-token-buffer-mib: 2.353806E+02 | global_load_balancing_loss: 1.000294E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11218.155 | params rms: 0.060743 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:44:00.545427 | finish at 2026-10-17 20:14:51 + [2026-10-12 08:34:39] iteration 298/ 2384 | consumed samples: 9536 | elapsed time per iteration (ms): 228042.5 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.09% | FLOPs estimator: workload-aware | logical tokens/s: 73570.5 | valid tokens/s: 73570.5 | physical tokens/s: 74203.1 | loss tokens/s: 60866.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.439851E-01 | lm-global-supervised-tokens: 1.388010E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.304335E-02 | lm-token-buffer-mib: 2.341622E+02 | global_load_balancing_loss: 9.994999E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11218.102 | params rms: 0.060742 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:08:16.668468 | finish at 2026-10-17 20:42:55 + [2026-10-12 08:38:21] iteration 299/ 2384 | consumed samples: 9568 | elapsed time per iteration (ms): 222175.6 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75513.3 | valid tokens/s: 75513.3 | physical tokens/s: 76052.0 | loss tokens/s: 57204.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.190895E-01 | lm-global-supervised-tokens: 1.270943E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.284477E-02 | lm-token-buffer-mib: 2.346425E+02 | global_load_balancing_loss: 1.001348E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11218.051 | params rms: 0.060742 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:40:36.229008 | finish at 2026-10-17 17:18:57 + [2026-10-12 08:42:24] iteration 300/ 2384 | consumed samples: 9600 | elapsed time per iteration (ms): 243639.0 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.30% | FLOPs estimator: workload-aware | logical tokens/s: 68860.7 | valid tokens/s: 68860.7 | physical tokens/s: 69491.1 | loss tokens/s: 52628.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.731886E-01 | lm-global-supervised-tokens: 1.282233E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.063171E-02 | lm-token-buffer-mib: 2.325250E+02 | global_load_balancing_loss: 9.982749E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11218.000 | params rms: 0.060742 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:02:23.755868 | finish at 2026-10-18 05:44:48 + [2026-10-12 08:46:04] iteration 301/ 2384 | consumed samples: 9632 | elapsed time per iteration (ms): 219875.0 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 76303.3 | valid tokens/s: 76303.3 | physical tokens/s: 76928.1 | loss tokens/s: 59392.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.988562E-01 | lm-global-supervised-tokens: 1.305893E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.919703E-02 | lm-token-buffer-mib: 2.336985E+02 | global_load_balancing_loss: 9.991685E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11217.948 | params rms: 0.060742 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:13:19.627980 | finish at 2026-10-17 15:59:24 + [2026-10-12 08:49:47] iteration 302/ 2384 | consumed samples: 9664 | elapsed time per iteration (ms): 222714.0 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.21% | FLOPs estimator: workload-aware | logical tokens/s: 75330.6 | valid tokens/s: 75330.6 | physical tokens/s: 76307.5 | loss tokens/s: 57845.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.268192E-01 | lm-global-supervised-tokens: 1.288300E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.297602E-02 | lm-token-buffer-mib: 2.338423E+02 | global_load_balancing_loss: 9.991417E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11217.898 | params rms: 0.060741 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:48:10.644768 | finish at 2026-10-17 17:37:58 + [2026-10-12 08:53:26] iteration 303/ 2384 | consumed samples: 9696 | elapsed time per iteration (ms): 218942.8 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 76628.1 | valid tokens/s: 76628.1 | physical tokens/s: 77125.3 | loss tokens/s: 58476.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.562861E-01 | lm-global-supervised-tokens: 1.280300E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.292962E-02 | lm-token-buffer-mib: 2.330835E+02 | global_load_balancing_loss: 1.000810E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11217.850 | params rms: 0.060741 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:33:39.888998 | finish at 2026-10-17 15:27:06 + [2026-10-12 08:57:09] iteration 304/ 2384 | consumed samples: 9728 | elapsed time per iteration (ms): 222609.7 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 75366.0 | valid tokens/s: 75366.0 | physical tokens/s: 75685.2 | loss tokens/s: 57194.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.829112E-01 | lm-global-supervised-tokens: 1.273194E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.848867E-02 | lm-token-buffer-mib: 2.330169E+02 | global_load_balancing_loss: 1.002866E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11217.799 | params rms: 0.060741 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:37:08.261223 | finish at 2026-10-17 17:34:17 + [2026-10-12 09:00:50] iteration 305/ 2384 | consumed samples: 9760 | elapsed time per iteration (ms): 221188.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75850.3 | valid tokens/s: 75850.3 | physical tokens/s: 76539.2 | loss tokens/s: 57904.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.514814E-01 | lm-global-supervised-tokens: 1.280786E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.263595E-02 | lm-token-buffer-mib: 2.340012E+02 | global_load_balancing_loss: 1.001108E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11217.740 | params rms: 0.060740 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:44:10.925551 | finish at 2026-10-17 16:45:01 + [2026-10-12 09:04:33] iteration 306/ 2384 | consumed samples: 9792 | elapsed time per iteration (ms): 223510.9 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 6.01% | FLOPs estimator: workload-aware | logical tokens/s: 75062.1 | valid tokens/s: 75062.1 | physical tokens/s: 76112.5 | loss tokens/s: 61948.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.896793E-01 | lm-global-supervised-tokens: 1.384612E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.109336E-02 | lm-token-buffer-mib: 2.361244E+02 | global_load_balancing_loss: 1.001217E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11217.680 | params rms: 0.060740 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:00:55.693841 | finish at 2026-10-17 18:05:29 + [2026-10-12 09:08:20] iteration 307/ 2384 | consumed samples: 9824 | elapsed time per iteration (ms): 226485.9 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 74075.9 | valid tokens/s: 74075.9 | physical tokens/s: 74480.0 | loss tokens/s: 56976.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.375826E-01 | lm-global-supervised-tokens: 1.290426E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.850001E-02 | lm-token-buffer-mib: 2.336607E+02 | global_load_balancing_loss: 1.001928E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11217.625 | params rms: 0.060740 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:40:11.197803 | finish at 2026-10-17 19:48:31 + [2026-10-12 09:12:02] iteration 308/ 2384 | consumed samples: 9856 | elapsed time per iteration (ms): 222253.3 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.09% | FLOPs estimator: workload-aware | logical tokens/s: 75486.8 | valid tokens/s: 75486.8 | physical tokens/s: 76153.8 | loss tokens/s: 58579.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.122817E-01 | lm-global-supervised-tokens: 1.301954E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.571169E-02 | lm-token-buffer-mib: 2.347978E+02 | global_load_balancing_loss: 9.999895E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11217.571 | params rms: 0.060739 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:09:57.928408 | finish at 2026-10-17 17:22:00 + [2026-10-12 09:15:45] iteration 309/ 2384 | consumed samples: 9888 | elapsed time per iteration (ms): 222971.1 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75243.9 | valid tokens/s: 75243.9 | physical tokens/s: 76511.1 | loss tokens/s: 56589.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.380052E-01 | lm-global-supervised-tokens: 1.261785E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.853641E-02 | lm-token-buffer-mib: 2.356678E+02 | global_load_balancing_loss: 9.994983E-01 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11217.513 | params rms: 0.060739 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:31:05.101457 | finish at 2026-10-17 17:46:50 + [2026-10-12 09:19:53] iteration 310/ 2384 | consumed samples: 9920 | elapsed time per iteration (ms): 247547.8 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 67773.6 | valid tokens/s: 67773.6 | physical tokens/s: 68631.2 | loss tokens/s: 53511.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.969408E-01 | lm-global-supervised-tokens: 1.324663E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.675149E-02 | lm-token-buffer-mib: 2.349331E+02 | global_load_balancing_loss: 1.001826E+00 | loss scale: 1.0 | grad norm: 0.205 | params norm: 11217.454 | params rms: 0.060739 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:36:54.116565 | finish at 2026-10-18 07:56:47 + [2026-10-12 09:23:36] iteration 311/ 2384 | consumed samples: 9952 | elapsed time per iteration (ms): 223472.2 | throughput per GPU (TFLOP/s/GPU): 26.9 | MFU 8.62% | FLOPs estimator: workload-aware | logical tokens/s: 75075.0 | valid tokens/s: 75075.0 | physical tokens/s: 75640.2 | loss tokens/s: 55094.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.013293E-01 | lm-global-supervised-tokens: 1.231200E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.512752E-02 | lm-token-buffer-mib: 2.343840E+02 | global_load_balancing_loss: 1.000428E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11217.397 | params rms: 0.060739 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:40:57.882784 | finish at 2026-10-17 18:04:34 + [2026-10-12 09:27:20] iteration 312/ 2384 | consumed samples: 9984 | elapsed time per iteration (ms): 223657.3 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75013.0 | valid tokens/s: 75013.0 | physical tokens/s: 75720.4 | loss tokens/s: 58232.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.502693E-01 | lm-global-supervised-tokens: 1.302414E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.647645E-02 | lm-token-buffer-mib: 2.327580E+02 | global_load_balancing_loss: 1.001010E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11217.340 | params rms: 0.060738 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:43:37.948809 | finish at 2026-10-17 18:10:58 + [2026-10-12 09:31:04] iteration 313/ 2384 | consumed samples: 10016 | elapsed time per iteration (ms): 224021.6 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 74890.9 | valid tokens/s: 74890.9 | physical tokens/s: 75969.2 | loss tokens/s: 56559.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.114722E-01 | lm-global-supervised-tokens: 1.267046E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.619019E-02 | lm-token-buffer-mib: 2.355015E+02 | global_load_balancing_loss: 1.000207E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11217.286 | params rms: 0.060738 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:52:28.727691 | finish at 2026-10-17 18:23:32 + [2026-10-12 09:34:45] iteration 314/ 2384 | consumed samples: 10048 | elapsed time per iteration (ms): 221045.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75899.3 | valid tokens/s: 75899.3 | physical tokens/s: 76524.5 | loss tokens/s: 62778.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.768421E-01 | lm-global-supervised-tokens: 1.387689E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.607935E-02 | lm-token-buffer-mib: 2.338562E+02 | global_load_balancing_loss: 9.997768E-01 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11217.231 | params rms: 0.060738 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:06:04.171758 | finish at 2026-10-17 16:40:49 + [2026-10-12 09:38:27] iteration 315/ 2384 | consumed samples: 10080 | elapsed time per iteration (ms): 222046.5 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 75557.1 | valid tokens/s: 75557.1 | physical tokens/s: 76279.8 | loss tokens/s: 55798.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.495719E-01 | lm-global-supervised-tokens: 1.238981E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.208408E-02 | lm-token-buffer-mib: 2.336317E+02 | global_load_balancing_loss: 1.000542E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11217.174 | params rms: 0.060737 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:36:54.117175 | finish at 2026-10-17 17:15:21 + [2026-10-12 09:42:08] iteration 316/ 2384 | consumed samples: 10112 | elapsed time per iteration (ms): 221382.7 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 75783.7 | valid tokens/s: 75783.7 | physical tokens/s: 76475.9 | loss tokens/s: 58704.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.253346E-01 | lm-global-supervised-tokens: 1.299608E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.548048E-02 | lm-token-buffer-mib: 2.344858E+02 | global_load_balancing_loss: 9.998728E-01 | loss scale: 1.0 | grad norm: 0.231 | params norm: 11217.116 | params rms: 0.060737 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:10:19.386552 | finish at 2026-10-17 16:52:28 + [2026-10-12 09:45:49] iteration 317/ 2384 | consumed samples: 10144 | elapsed time per iteration (ms): 220911.2 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75945.3 | valid tokens/s: 75945.3 | physical tokens/s: 76831.7 | loss tokens/s: 56717.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.057788E-01 | lm-global-supervised-tokens: 1.252949E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.677446E-02 | lm-token-buffer-mib: 2.347129E+02 | global_load_balancing_loss: 9.994874E-01 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11217.057 | params rms: 0.060737 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:50:23.466266 | finish at 2026-10-17 16:36:13 + [2026-10-12 09:49:31] iteration 318/ 2384 | consumed samples: 10176 | elapsed time per iteration (ms): 221517.4 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 75737.5 | valid tokens/s: 75737.5 | physical tokens/s: 76407.7 | loss tokens/s: 58422.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.950673E-01 | lm-global-supervised-tokens: 1.294155E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.280205E-02 | lm-token-buffer-mib: 2.348557E+02 | global_load_balancing_loss: 1.001084E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11216.996 | params rms: 0.060736 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:07:34.846984 | finish at 2026-10-17 16:57:05 + [2026-10-12 09:53:12] iteration 319/ 2384 | consumed samples: 10208 | elapsed time per iteration (ms): 221370.4 | throughput per GPU (TFLOP/s/GPU): 28.0 | MFU 8.99% | FLOPs estimator: workload-aware | logical tokens/s: 75787.7 | valid tokens/s: 75787.7 | physical tokens/s: 76415.6 | loss tokens/s: 56164.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 9.268942E-01 | lm-global-supervised-tokens: 1.243309E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.696726E-02 | lm-token-buffer-mib: 2.324145E+02 | global_load_balancing_loss: 1.002020E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11216.934 | params rms: 0.060736 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:58:49.951228 | finish at 2026-10-17 16:52:02 + [2026-10-12 09:57:12] iteration 320/ 2384 | consumed samples: 10240 | elapsed time per iteration (ms): 240348.2 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 69803.7 | valid tokens/s: 69803.7 | physical tokens/s: 70103.6 | loss tokens/s: 50463.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.464275E-01 | lm-global-supervised-tokens: 1.212876E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.744806E-02 | lm-token-buffer-mib: 2.329696E+02 | global_load_balancing_loss: 1.001921E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11216.874 | params rms: 0.060736 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:47:58.719910 | finish at 2026-10-18 03:45:11 +times across ranks (ms): + save-checkpoint: + rank 0: 25750.11 + rank 1: 25750.07 + rank 2: 25750.08 + rank 3: 25750.08 + rank 4: 25750.08 + rank 5: 25750.08 + rank 6: 25750.07 + rank 7: 25750.13 + rank 8: 25750.05 + rank 9: 25750.08 + rank 10: 25750.03 + rank 11: 25749.98 + rank 12: 25749.99 + rank 13: 25749.97 + rank 14: 25750.00 + rank 15: 25749.97 + rank 16: 25750.07 + rank 17: 25750.09 + rank 18: 25750.12 + rank 19: 25750.14 + rank 20: 25750.07 + rank 21: 25750.07 + rank 22: 25750.13 + rank 23: 25750.08 + rank 24: 25750.20 + rank 25: 25750.20 + rank 26: 25750.11 + rank 27: 25750.14 + rank 28: 25749.92 + rank 29: 25750.21 + rank 30: 25750.23 + rank 31: 25750.14 + rank 32: 25750.47 + rank 33: 25750.42 + rank 34: 25750.45 + rank 35: 25750.37 + rank 36: 25750.39 + rank 37: 25750.42 + rank 38: 25750.39 + rank 39: 25750.39 + rank 40: 25750.14 + rank 41: 25750.13 + rank 42: 25750.12 + rank 43: 25750.13 + rank 44: 25750.12 + rank 45: 25750.11 + rank 46: 25750.10 + rank 47: 25750.13 + rank 48: 25750.81 + rank 49: 25750.91 + rank 50: 25750.85 + rank 51: 25751.05 + rank 52: 25750.83 + rank 53: 25750.83 + rank 54: 25750.83 + rank 55: 25750.83 + rank 56: 25750.23 + rank 57: 25750.29 + rank 58: 25750.29 + rank 59: 25750.29 + rank 60: 25750.28 + rank 61: 25750.23 + rank 62: 25750.25 + rank 63: 25750.21 + [2026-10-12 10:01:22] iteration 321/ 2384 | consumed samples: 10272 | elapsed time per iteration (ms): 223589.3 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75035.7 | valid tokens/s: 75035.7 | physical tokens/s: 75682.8 | loss tokens/s: 55888.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.246952E-01 | lm-global-supervised-tokens: 1.249607E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.439856E-02 | lm-token-buffer-mib: 2.331747E+02 | global_load_balancing_loss: 1.000327E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11216.812 | params rms: 0.060735 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:07:44.733599 | finish at 2026-10-17 18:09:06 + [2026-10-12 10:05:09] iteration 322/ 2384 | consumed samples: 10304 | elapsed time per iteration (ms): 227657.8 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 73694.8 | valid tokens/s: 73694.8 | physical tokens/s: 74154.0 | loss tokens/s: 59975.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.792954E-01 | lm-global-supervised-tokens: 1.365388E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.985421E-02 | lm-token-buffer-mib: 2.335993E+02 | global_load_balancing_loss: 1.000519E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11216.754 | params rms: 0.060735 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:23:50.462175 | finish at 2026-10-17 20:29:00 + [2026-10-12 10:08:50] iteration 323/ 2384 | consumed samples: 10336 | elapsed time per iteration (ms): 220504.4 | throughput per GPU (TFLOP/s/GPU): 27.7 | MFU 8.88% | FLOPs estimator: workload-aware | logical tokens/s: 76085.4 | valid tokens/s: 76085.4 | physical tokens/s: 76719.8 | loss tokens/s: 61638.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.981634E-01 | lm-global-supervised-tokens: 1.359162E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.671519E-02 | lm-token-buffer-mib: 2.332242E+02 | global_load_balancing_loss: 9.993498E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11216.697 | params rms: 0.060735 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:14:19.666215 | finish at 2026-10-17 16:23:09 + [2026-10-12 10:12:32] iteration 324/ 2384 | consumed samples: 10368 | elapsed time per iteration (ms): 221729.6 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.21% | FLOPs estimator: workload-aware | logical tokens/s: 75665.2 | valid tokens/s: 75665.2 | physical tokens/s: 76182.7 | loss tokens/s: 56433.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.724797E-01 | lm-global-supervised-tokens: 1.251293E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.937678E-02 | lm-token-buffer-mib: 2.340467E+02 | global_load_balancing_loss: 9.992922E-01 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11216.642 | params rms: 0.060734 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:52:43.039751 | finish at 2026-10-17 17:05:15 + [2026-10-12 10:16:12] iteration 325/ 2384 | consumed samples: 10400 | elapsed time per iteration (ms): 220368.7 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 76132.3 | valid tokens/s: 76132.3 | physical tokens/s: 76542.3 | loss tokens/s: 56947.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.704823E-01 | lm-global-supervised-tokens: 1.254935E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.340093E-02 | lm-token-buffer-mib: 2.330190E+02 | global_load_balancing_loss: 1.002173E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11216.591 | params rms: 0.060734 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:02:19.159738 | finish at 2026-10-17 16:18:31 + [2026-10-12 10:19:55] iteration 326/ 2384 | consumed samples: 10432 | elapsed time per iteration (ms): 222963.2 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 75246.5 | valid tokens/s: 75246.5 | physical tokens/s: 75760.7 | loss tokens/s: 61033.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.321286E-01 | lm-global-supervised-tokens: 1.360829E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.752356E-02 | lm-token-buffer-mib: 2.339334E+02 | global_load_balancing_loss: 1.000915E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11216.541 | params rms: 0.060734 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:27:38.199550 | finish at 2026-10-17 17:47:33 + [2026-10-12 10:23:43] iteration 327/ 2384 | consumed samples: 10464 | elapsed time per iteration (ms): 227856.9 | throughput per GPU (TFLOP/s/GPU): 26.8 | MFU 8.58% | FLOPs estimator: workload-aware | logical tokens/s: 73630.3 | valid tokens/s: 73630.3 | physical tokens/s: 74447.0 | loss tokens/s: 57873.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.053601E-01 | lm-global-supervised-tokens: 1.318696E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.649303E-02 | lm-token-buffer-mib: 2.335292E+02 | global_load_balancing_loss: 9.990826E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11216.491 | params rms: 0.060734 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:11:41.630991 | finish at 2026-10-17 20:35:24 + [2026-10-12 10:27:26] iteration 328/ 2384 | consumed samples: 10496 | elapsed time per iteration (ms): 223008.8 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.82% | FLOPs estimator: workload-aware | logical tokens/s: 75231.0 | valid tokens/s: 75231.0 | physical tokens/s: 75827.8 | loss tokens/s: 61570.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.846053E-01 | lm-global-supervised-tokens: 1.373080E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.453076E-02 | lm-token-buffer-mib: 2.347976E+02 | global_load_balancing_loss: 1.001653E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11216.441 | params rms: 0.060733 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:21:46.181585 | finish at 2026-10-17 17:49:12 + [2026-10-12 10:31:07] iteration 329/ 2384 | consumed samples: 10528 | elapsed time per iteration (ms): 221055.0 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75895.9 | valid tokens/s: 75895.9 | physical tokens/s: 76445.3 | loss tokens/s: 53231.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.842802E-01 | lm-global-supervised-tokens: 1.176699E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.729044E-02 | lm-token-buffer-mib: 2.339424E+02 | global_load_balancing_loss: 1.000646E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11216.392 | params rms: 0.060733 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:11:07.963403 | finish at 2026-10-17 16:42:15 + [2026-10-12 10:35:15] iteration 330/ 2384 | consumed samples: 10560 | elapsed time per iteration (ms): 248683.1 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 67464.1 | valid tokens/s: 67464.1 | physical tokens/s: 68073.2 | loss tokens/s: 52183.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.424689E-01 | lm-global-supervised-tokens: 1.297722E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.732664E-02 | lm-token-buffer-mib: 2.334445E+02 | global_load_balancing_loss: 1.000647E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11216.346 | params rms: 0.060733 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:53:15.026155 | finish at 2026-10-18 08:28:31 + [2026-10-12 10:39:00] iteration 331/ 2384 | consumed samples: 10592 | elapsed time per iteration (ms): 224707.1 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 74662.4 | valid tokens/s: 74662.4 | physical tokens/s: 75562.9 | loss tokens/s: 54331.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.099788E-01 | lm-global-supervised-tokens: 1.220860E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.740181E-02 | lm-token-buffer-mib: 2.332040E+02 | global_load_balancing_loss: 1.000265E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11216.300 | params rms: 0.060733 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:08:43.710877 | finish at 2026-10-17 18:47:44 + [2026-10-12 10:42:41] iteration 332/ 2384 | consumed samples: 10624 | elapsed time per iteration (ms): 221134.4 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.65% | FLOPs estimator: workload-aware | logical tokens/s: 75868.7 | valid tokens/s: 75868.7 | physical tokens/s: 76439.9 | loss tokens/s: 53652.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.483653E-01 | lm-global-supervised-tokens: 1.186432E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.357440E-02 | lm-token-buffer-mib: 2.340215E+02 | global_load_balancing_loss: 1.001064E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11216.253 | params rms: 0.060732 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:02:47.840435 | finish at 2026-10-17 16:45:29 + [2026-10-12 10:46:23] iteration 333/ 2384 | consumed samples: 10656 | elapsed time per iteration (ms): 221756.7 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 75655.9 | valid tokens/s: 75655.9 | physical tokens/s: 75935.0 | loss tokens/s: 52187.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.773472E-01 | lm-global-supervised-tokens: 1.157282E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.487178E-02 | lm-token-buffer-mib: 2.329706E+02 | global_load_balancing_loss: 1.001186E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11216.208 | params rms: 0.060732 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:20:22.990802 | finish at 2026-10-17 17:06:46 + [2026-10-12 10:50:05] iteration 334/ 2384 | consumed samples: 10688 | elapsed time per iteration (ms): 221970.3 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 75583.2 | valid tokens/s: 75583.2 | physical tokens/s: 75942.7 | loss tokens/s: 56764.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.744206E-01 | lm-global-supervised-tokens: 1.259992E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.280350E-02 | lm-token-buffer-mib: 2.330455E+02 | global_load_balancing_loss: 1.001116E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11216.163 | params rms: 0.060732 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:23:59.120293 | finish at 2026-10-17 17:14:04 + [2026-10-12 10:53:50] iteration 335/ 2384 | consumed samples: 10720 | elapsed time per iteration (ms): 224470.9 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 74741.1 | valid tokens/s: 74741.1 | physical tokens/s: 75438.9 | loss tokens/s: 50647.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.449394E-01 | lm-global-supervised-tokens: 1.136881E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.469065E-02 | lm-token-buffer-mib: 2.337703E+02 | global_load_balancing_loss: 1.000582E+00 | loss scale: 1.0 | grad norm: 0.425 | params norm: 11216.121 | params rms: 0.060732 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:45:40.786287 | finish at 2026-10-17 18:39:30 + [2026-10-12 10:57:35] iteration 336/ 2384 | consumed samples: 10752 | elapsed time per iteration (ms): 225351.8 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.68% | FLOPs estimator: workload-aware | logical tokens/s: 74448.9 | valid tokens/s: 74448.9 | physical tokens/s: 74876.1 | loss tokens/s: 56993.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.047768E-01 | lm-global-supervised-tokens: 1.284358E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.571272E-02 | lm-token-buffer-mib: 2.336034E+02 | global_load_balancing_loss: 1.000391E+00 | loss scale: 1.0 | grad norm: 0.286 | params norm: 11216.079 | params rms: 0.060731 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:12:00.401855 | finish at 2026-10-17 19:09:35 + [2026-10-12 11:01:17] iteration 337/ 2384 | consumed samples: 10784 | elapsed time per iteration (ms): 222432.4 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.87% | FLOPs estimator: workload-aware | logical tokens/s: 75425.9 | valid tokens/s: 75425.8 | physical tokens/s: 76163.6 | loss tokens/s: 57540.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.754597E-01 | lm-global-supervised-tokens: 1.279885E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.800762E-02 | lm-token-buffer-mib: 2.340045E+02 | global_load_balancing_loss: 1.001455E+00 | loss scale: 1.0 | grad norm: 0.325 | params norm: 11216.034 | params rms: 0.060731 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:28:39.194518 | finish at 2026-10-17 17:29:57 + [2026-10-12 11:04:59] iteration 338/ 2384 | consumed samples: 10816 | elapsed time per iteration (ms): 221300.1 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 75811.8 | valid tokens/s: 75811.8 | physical tokens/s: 76375.4 | loss tokens/s: 61096.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.461739E-01 | lm-global-supervised-tokens: 1.352056E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.910433E-02 | lm-token-buffer-mib: 2.345024E+02 | global_load_balancing_loss: 1.000418E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11215.989 | params rms: 0.060731 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:46:20.103805 | finish at 2026-10-17 16:51:19 + [2026-10-12 11:08:39] iteration 339/ 2384 | consumed samples: 10848 | elapsed time per iteration (ms): 220276.3 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.02% | FLOPs estimator: workload-aware | logical tokens/s: 76164.4 | valid tokens/s: 76164.4 | physical tokens/s: 76968.1 | loss tokens/s: 59797.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.739499E-01 | lm-global-supervised-tokens: 1.317187E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.053146E-02 | lm-token-buffer-mib: 2.348017E+02 | global_load_balancing_loss: 1.000296E+00 | loss scale: 1.0 | grad norm: 0.374 | params norm: 11215.944 | params rms: 0.060731 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:07:44.951981 | finish at 2026-10-17 16:16:24 + [2026-10-12 11:12:41] iteration 340/ 2384 | consumed samples: 10880 | elapsed time per iteration (ms): 242229.8 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 69261.5 | valid tokens/s: 69261.5 | physical tokens/s: 69679.0 | loss tokens/s: 52213.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.268649E-01 | lm-global-supervised-tokens: 1.264775E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.488287E-02 | lm-token-buffer-mib: 2.330031E+02 | global_load_balancing_loss: 1.000797E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11215.898 | params rms: 0.060730 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:31:57.732126 | finish at 2026-10-18 04:44:39 + [2026-10-12 11:16:23] iteration 341/ 2384 | consumed samples: 10912 | elapsed time per iteration (ms): 222183.6 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.95% | FLOPs estimator: workload-aware | logical tokens/s: 75510.4 | valid tokens/s: 75510.4 | physical tokens/s: 76244.3 | loss tokens/s: 59381.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.435949E-01 | lm-global-supervised-tokens: 1.319357E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.654151E-02 | lm-token-buffer-mib: 2.362055E+02 | global_load_balancing_loss: 9.991992E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11215.852 | params rms: 0.060730 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:05:21.124408 | finish at 2026-10-17 17:21:44 + [2026-10-12 11:20:07] iteration 342/ 2384 | consumed samples: 10944 | elapsed time per iteration (ms): 223363.6 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 75111.5 | valid tokens/s: 75111.5 | physical tokens/s: 75835.2 | loss tokens/s: 52596.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.135239E-01 | lm-global-supervised-tokens: 1.174809E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.812387E-02 | lm-token-buffer-mib: 2.357536E+02 | global_load_balancing_loss: 9.995120E-01 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11215.807 | params rms: 0.060730 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:41:48.375809 | finish at 2026-10-17 18:01:55 + [2026-10-12 11:23:48] iteration 343/ 2384 | consumed samples: 10976 | elapsed time per iteration (ms): 221215.2 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 75841.1 | valid tokens/s: 75841.1 | physical tokens/s: 76741.8 | loss tokens/s: 57784.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.429739E-01 | lm-global-supervised-tokens: 1.278290E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.494522E-02 | lm-token-buffer-mib: 2.333981E+02 | global_load_balancing_loss: 1.001796E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11215.762 | params rms: 0.060730 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:25:00.151074 | finish at 2026-10-17 16:48:48 + [2026-10-12 11:27:29] iteration 344/ 2384 | consumed samples: 11008 | elapsed time per iteration (ms): 221503.4 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 75742.4 | valid tokens/s: 75742.3 | physical tokens/s: 76304.2 | loss tokens/s: 60940.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.884207E-01 | lm-global-supervised-tokens: 1.349852E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.038342E-02 | lm-token-buffer-mib: 2.342478E+02 | global_load_balancing_loss: 1.001737E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11215.716 | params rms: 0.060729 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:31:06.929369 | finish at 2026-10-17 16:58:36 + [2026-10-12 11:31:13] iteration 345/ 2384 | consumed samples: 11040 | elapsed time per iteration (ms): 223153.6 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.68% | FLOPs estimator: workload-aware | logical tokens/s: 75182.2 | valid tokens/s: 75182.2 | physical tokens/s: 75643.5 | loss tokens/s: 52219.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.136949E-01 | lm-global-supervised-tokens: 1.165303E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.361282E-02 | lm-token-buffer-mib: 2.328493E+02 | global_load_balancing_loss: 1.001033E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11215.669 | params rms: 0.060729 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:23:30.242371 | finish at 2026-10-17 17:54:43 + [2026-10-12 11:34:55] iteration 346/ 2384 | consumed samples: 11072 | elapsed time per iteration (ms): 222564.8 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 75381.2 | valid tokens/s: 75381.2 | physical tokens/s: 76218.9 | loss tokens/s: 57853.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.196497E-01 | lm-global-supervised-tokens: 1.287620E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.101409E-02 | lm-token-buffer-mib: 2.345565E+02 | global_load_balancing_loss: 9.986954E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11215.622 | params rms: 0.060729 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:59:47.147481 | finish at 2026-10-17 17:34:42 + [2026-10-12 11:38:39] iteration 347/ 2384 | consumed samples: 11104 | elapsed time per iteration (ms): 224076.4 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.83% | FLOPs estimator: workload-aware | logical tokens/s: 74872.5 | valid tokens/s: 74872.5 | physical tokens/s: 75518.8 | loss tokens/s: 55052.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.606418E-01 | lm-global-supervised-tokens: 1.233589E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.429310E-02 | lm-token-buffer-mib: 2.354793E+02 | global_load_balancing_loss: 1.000781E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11215.569 | params rms: 0.060729 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:47:23.696820 | finish at 2026-10-17 18:26:03 + [2026-10-12 11:42:21] iteration 348/ 2384 | consumed samples: 11136 | elapsed time per iteration (ms): 222097.6 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.90% | FLOPs estimator: workload-aware | logical tokens/s: 75539.6 | valid tokens/s: 75539.6 | physical tokens/s: 75696.6 | loss tokens/s: 54626.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.182152E-01 | lm-global-supervised-tokens: 1.213242E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.793762E-02 | lm-token-buffer-mib: 2.327006E+02 | global_load_balancing_loss: 1.001630E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11215.512 | params rms: 0.060728 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:36:30.709682 | finish at 2026-10-17 17:18:52 + [2026-10-12 11:46:04] iteration 349/ 2384 | consumed samples: 11168 | elapsed time per iteration (ms): 222374.5 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 75445.7 | valid tokens/s: 75445.7 | physical tokens/s: 76324.7 | loss tokens/s: 55915.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.624783E-01 | lm-global-supervised-tokens: 1.243424E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.314168E-02 | lm-token-buffer-mib: 2.358425E+02 | global_load_balancing_loss: 1.001403E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11215.456 | params rms: 0.060728 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:42:12.126496 | finish at 2026-10-17 17:28:16 + [2026-10-12 11:50:12] iteration 350/ 2384 | consumed samples: 11200 | elapsed time per iteration (ms): 247855.2 | throughput per GPU (TFLOP/s/GPU): 17.6 | MFU 5.63% | FLOPs estimator: workload-aware | logical tokens/s: 67689.5 | valid tokens/s: 67689.5 | physical tokens/s: 69024.3 | loss tokens/s: 48413.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.977352E-01 | lm-global-supervised-tokens: 1.199947E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.757517E-02 | lm-token-buffer-mib: 2.375336E+02 | global_load_balancing_loss: 1.000691E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11215.403 | params rms: 0.060728 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:02:17.520066 | finish at 2026-10-18 07:52:29 + [2026-10-12 11:53:55] iteration 351/ 2384 | consumed samples: 11232 | elapsed time per iteration (ms): 223924.0 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.45% | FLOPs estimator: workload-aware | logical tokens/s: 74923.6 | valid tokens/s: 74923.6 | physical tokens/s: 76050.4 | loss tokens/s: 56114.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.394841E-01 | lm-global-supervised-tokens: 1.256540E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.262165E-02 | lm-token-buffer-mib: 2.353889E+02 | global_load_balancing_loss: 9.991482E-01 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11215.351 | params rms: 0.060727 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:27:17.474116 | finish at 2026-10-17 18:21:13 + [2026-10-12 11:57:36] iteration 352/ 2384 | consumed samples: 11264 | elapsed time per iteration (ms): 220078.3 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 76232.9 | valid tokens/s: 76232.9 | physical tokens/s: 76949.8 | loss tokens/s: 55921.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.156951E-01 | lm-global-supervised-tokens: 1.230707E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.096101E-02 | lm-token-buffer-mib: 2.340786E+02 | global_load_balancing_loss: 9.990356E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11215.297 | params rms: 0.060727 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:13:19.186504 | finish at 2026-10-17 16:10:55 + [2026-10-12 12:01:16] iteration 353/ 2384 | consumed samples: 11296 | elapsed time per iteration (ms): 220575.0 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 76061.2 | valid tokens/s: 76061.2 | physical tokens/s: 76598.6 | loss tokens/s: 57297.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.802509E-01 | lm-global-supervised-tokens: 1.263837E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.853776E-02 | lm-token-buffer-mib: 2.342584E+02 | global_load_balancing_loss: 9.997311E-01 | loss scale: 1.0 | grad norm: 0.255 | params norm: 11215.241 | params rms: 0.060727 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:26:27.745199 | finish at 2026-10-17 16:27:44 + [2026-10-12 12:04:55] iteration 354/ 2384 | consumed samples: 11328 | elapsed time per iteration (ms): 219076.6 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 76581.4 | valid tokens/s: 76581.4 | physical tokens/s: 76954.0 | loss tokens/s: 56384.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.119102E-01 | lm-global-supervised-tokens: 1.235254E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.326328E-02 | lm-token-buffer-mib: 2.324739E+02 | global_load_balancing_loss: 9.983933E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11215.189 | params rms: 0.060727 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:32:05.526707 | finish at 2026-10-17 15:37:01 + [2026-10-12 12:08:36] iteration 355/ 2384 | consumed samples: 11360 | elapsed time per iteration (ms): 220422.5 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 76113.7 | valid tokens/s: 76113.7 | physical tokens/s: 76981.5 | loss tokens/s: 61068.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.180293E-01 | lm-global-supervised-tokens: 1.346083E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.557821E-02 | lm-token-buffer-mib: 2.333697E+02 | global_load_balancing_loss: 1.000232E+00 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11215.136 | params rms: 0.060726 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:13:57.258092 | finish at 2026-10-17 16:22:33 + [2026-10-12 12:12:20] iteration 356/ 2384 | consumed samples: 11392 | elapsed time per iteration (ms): 224123.4 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.90% | FLOPs estimator: workload-aware | logical tokens/s: 74856.9 | valid tokens/s: 74856.9 | physical tokens/s: 75398.5 | loss tokens/s: 55865.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.108613E-01 | lm-global-supervised-tokens: 1.252086E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.763425E-02 | lm-token-buffer-mib: 2.348187E+02 | global_load_balancing_loss: 1.003325E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11215.079 | params rms: 0.060726 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:15:22.189871 | finish at 2026-10-17 18:27:42 + [2026-10-12 12:16:04] iteration 357/ 2384 | consumed samples: 11424 | elapsed time per iteration (ms): 224732.4 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.96% | FLOPs estimator: workload-aware | logical tokens/s: 74654.1 | valid tokens/s: 74654.1 | physical tokens/s: 75425.7 | loss tokens/s: 53141.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.325652E-01 | lm-global-supervised-tokens: 1.194251E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.439008E-02 | lm-token-buffer-mib: 2.357594E+02 | global_load_balancing_loss: 1.003130E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11215.022 | params rms: 0.060726 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:32:12.624457 | finish at 2026-10-17 18:48:17 + [2026-10-12 12:19:45] iteration 358/ 2384 | consumed samples: 11456 | elapsed time per iteration (ms): 220253.3 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 76172.1 | valid tokens/s: 76172.1 | physical tokens/s: 76822.1 | loss tokens/s: 57009.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.843102E-01 | lm-global-supervised-tokens: 1.255644E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.213092E-02 | lm-token-buffer-mib: 2.325840E+02 | global_load_balancing_loss: 1.000959E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11214.964 | params rms: 0.060725 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:57:13.103103 | finish at 2026-10-17 16:16:58 + [2026-10-12 12:23:31] iteration 359/ 2384 | consumed samples: 11488 | elapsed time per iteration (ms): 226134.2 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 74191.3 | valid tokens/s: 74191.3 | physical tokens/s: 74504.7 | loss tokens/s: 56851.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.346433E-01 | lm-global-supervised-tokens: 1.285602E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.622496E-02 | lm-token-buffer-mib: 2.329502E+02 | global_load_balancing_loss: 9.993150E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11214.906 | params rms: 0.060725 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:12:01.843547 | finish at 2026-10-17 19:35:33 + [2026-10-12 12:27:35] iteration 360/ 2384 | consumed samples: 11520 | elapsed time per iteration (ms): 244646.7 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.48% | FLOPs estimator: workload-aware | logical tokens/s: 68577.3 | valid tokens/s: 68577.3 | physical tokens/s: 69458.9 | loss tokens/s: 53327.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.579689E-01 | lm-global-supervised-tokens: 1.304638E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.452916E-02 | lm-token-buffer-mib: 2.352685E+02 | global_load_balancing_loss: 9.991683E-01 | loss scale: 1.0 | grad norm: 0.299 | params norm: 11214.849 | params rms: 0.060725 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:32:44.866562 | finish at 2026-10-18 06:00:20 +times across ranks (ms): + save-checkpoint: + rank 0: 24344.12 + rank 1: 24344.21 + rank 2: 24344.13 + rank 3: 24344.13 + rank 4: 24344.12 + rank 5: 24344.13 + rank 6: 24344.14 + rank 7: 24344.13 + rank 8: 24343.19 + rank 9: 24343.22 + rank 10: 24343.23 + rank 11: 24343.18 + rank 12: 24343.20 + rank 13: 24343.17 + rank 14: 24343.19 + rank 15: 24343.17 + rank 16: 24342.44 + rank 17: 24342.47 + rank 18: 24342.44 + rank 19: 24342.45 + rank 20: 24342.43 + rank 21: 24342.42 + rank 22: 24342.46 + rank 23: 24342.42 + rank 24: 24342.88 + rank 25: 24342.92 + rank 26: 24342.88 + rank 27: 24342.88 + rank 28: 24343.25 + rank 29: 24342.92 + rank 30: 24342.77 + rank 31: 24343.18 + rank 32: 24342.71 + rank 33: 24342.71 + rank 34: 24342.53 + rank 35: 24342.69 + rank 36: 24342.65 + rank 37: 24342.70 + rank 38: 24342.62 + rank 39: 24342.70 + rank 40: 24342.31 + rank 41: 24342.28 + rank 42: 24342.32 + rank 43: 24342.30 + rank 44: 24342.27 + rank 45: 24342.28 + rank 46: 24342.27 + rank 47: 24342.27 + rank 48: 24342.65 + rank 49: 24342.67 + rank 50: 24342.69 + rank 51: 24342.72 + rank 52: 24342.67 + rank 53: 24342.71 + rank 54: 24342.66 + rank 55: 24342.64 + rank 56: 24342.31 + rank 57: 24342.37 + rank 58: 24342.33 + rank 59: 24342.18 + rank 60: 24342.34 + rank 61: 24342.30 + rank 62: 24342.33 + rank 63: 24342.29 + [2026-10-12 12:31:47] iteration 361/ 2384 | consumed samples: 11552 | elapsed time per iteration (ms): 227446.9 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 73763.2 | valid tokens/s: 73763.2 | physical tokens/s: 74408.1 | loss tokens/s: 55485.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.925350E-01 | lm-global-supervised-tokens: 1.262001E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.761645E-02 | lm-token-buffer-mib: 2.335757E+02 | global_load_balancing_loss: 9.999867E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11214.792 | params rms: 0.060724 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:48:45.121406 | finish at 2026-10-17 20:20:32 + [2026-10-12 12:35:37] iteration 362/ 2384 | consumed samples: 11584 | elapsed time per iteration (ms): 230078.4 | throughput per GPU (TFLOP/s/GPU): 28.0 | MFU 8.96% | FLOPs estimator: workload-aware | logical tokens/s: 72919.3 | valid tokens/s: 72919.3 | physical tokens/s: 73898.7 | loss tokens/s: 54290.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.282366E-01 | lm-global-supervised-tokens: 1.249097E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.941124E-02 | lm-token-buffer-mib: 2.349002E+02 | global_load_balancing_loss: 1.000973E+00 | loss scale: 1.0 | grad norm: 0.506 | params norm: 11214.733 | params rms: 0.060724 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:13:38.534715 | finish at 2026-10-17 21:49:16 + [2026-10-12 12:39:18] iteration 363/ 2384 | consumed samples: 11616 | elapsed time per iteration (ms): 220678.7 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 76025.4 | valid tokens/s: 76025.4 | physical tokens/s: 76586.7 | loss tokens/s: 56309.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.564329E-01 | lm-global-supervised-tokens: 1.242639E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.470163E-02 | lm-token-buffer-mib: 2.328392E+02 | global_load_balancing_loss: 1.000116E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11214.671 | params rms: 0.060724 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:53:11.593855 | finish at 2026-10-17 16:32:30 + [2026-10-12 12:43:05] iteration 364/ 2384 | consumed samples: 11648 | elapsed time per iteration (ms): 227296.5 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.01% | FLOPs estimator: workload-aware | logical tokens/s: 73812.0 | valid tokens/s: 73812.0 | physical tokens/s: 74558.5 | loss tokens/s: 54666.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.599037E-01 | lm-global-supervised-tokens: 1.242540E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.212679E-02 | lm-token-buffer-mib: 2.344292E+02 | global_load_balancing_loss: 1.000800E+00 | loss scale: 1.0 | grad norm: 0.271 | params norm: 11214.610 | params rms: 0.060723 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:32:18.936195 | finish at 2026-10-17 20:15:24 + [2026-10-12 12:46:44] iteration 365/ 2384 | consumed samples: 11680 | elapsed time per iteration (ms): 219130.2 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 76562.6 | valid tokens/s: 76562.6 | physical tokens/s: 77164.1 | loss tokens/s: 57353.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.629952E-01 | lm-global-supervised-tokens: 1.256776E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.442973E-02 | lm-token-buffer-mib: 2.327145E+02 | global_load_balancing_loss: 9.996037E-01 | loss scale: 1.0 | grad norm: 0.181 | params norm: 11214.549 | params rms: 0.060723 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:53:43.868803 | finish at 2026-10-17 15:40:28 + [2026-10-12 12:50:27] iteration 366/ 2384 | consumed samples: 11712 | elapsed time per iteration (ms): 222178.2 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 75512.3 | valid tokens/s: 75512.3 | physical tokens/s: 76320.2 | loss tokens/s: 59453.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.957277E-01 | lm-global-supervised-tokens: 1.320924E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.571933E-02 | lm-token-buffer-mib: 2.350067E+02 | global_load_balancing_loss: 1.000143E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11214.487 | params rms: 0.060723 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:32:35.513312 | finish at 2026-10-17 17:23:02 + [2026-10-12 12:54:10] iteration 367/ 2384 | consumed samples: 11744 | elapsed time per iteration (ms): 223043.7 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.19% | FLOPs estimator: workload-aware | logical tokens/s: 75219.2 | valid tokens/s: 75219.2 | physical tokens/s: 75572.1 | loss tokens/s: 59289.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.256144E-01 | lm-global-supervised-tokens: 1.322406E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.809042E-02 | lm-token-buffer-mib: 2.330742E+02 | global_load_balancing_loss: 1.000600E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11214.425 | params rms: 0.060722 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:57:59.173636 | finish at 2026-10-17 17:52:09 + [2026-10-12 12:57:54] iteration 368/ 2384 | consumed samples: 11776 | elapsed time per iteration (ms): 224132.2 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 74854.1 | valid tokens/s: 74854.0 | physical tokens/s: 76013.7 | loss tokens/s: 57469.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.825384E-01 | lm-global-supervised-tokens: 1.288068E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.831271E-02 | lm-token-buffer-mib: 2.355363E+02 | global_load_balancing_loss: 1.001001E+00 | loss scale: 1.0 | grad norm: 0.534 | params norm: 11214.364 | params rms: 0.060722 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:30:50.465698 | finish at 2026-10-17 18:28:44 + [2026-10-12 13:01:39] iteration 369/ 2384 | consumed samples: 11808 | elapsed time per iteration (ms): 225347.6 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 74450.2 | valid tokens/s: 74450.2 | physical tokens/s: 75510.9 | loss tokens/s: 58548.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.013859E-01 | lm-global-supervised-tokens: 1.319383E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.030491E-02 | lm-token-buffer-mib: 2.357317E+02 | global_load_balancing_loss: 1.002165E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11214.301 | params rms: 0.060722 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:07:55.362082 | finish at 2026-10-17 19:09:35 + [2026-10-12 13:05:44] iteration 370/ 2384 | consumed samples: 11840 | elapsed time per iteration (ms): 244859.6 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.38% | FLOPs estimator: workload-aware | logical tokens/s: 68517.6 | valid tokens/s: 68517.6 | physical tokens/s: 69308.1 | loss tokens/s: 53327.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.930768E-01 | lm-global-supervised-tokens: 1.305778E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.294126E-02 | lm-token-buffer-mib: 2.363906E+02 | global_load_balancing_loss: 1.002388E+00 | loss scale: 1.0 | grad norm: 0.300 | params norm: 11214.237 | params rms: 0.060721 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:59:07.185077 | finish at 2026-10-18 06:04:51 + [2026-10-12 13:09:25] iteration 371/ 2384 | consumed samples: 11872 | elapsed time per iteration (ms): 221377.3 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 75785.5 | valid tokens/s: 75785.5 | physical tokens/s: 76429.7 | loss tokens/s: 57126.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.320919E-01 | lm-global-supervised-tokens: 1.264660E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.392703E-02 | lm-token-buffer-mib: 2.341680E+02 | global_load_balancing_loss: 1.001595E+00 | loss scale: 1.0 | grad norm: 0.310 | params norm: 11214.176 | params rms: 0.060721 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:47:12.534225 | finish at 2026-10-17 16:56:38 + [2026-10-12 13:13:08] iteration 372/ 2384 | consumed samples: 11904 | elapsed time per iteration (ms): 222808.4 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 75298.6 | valid tokens/s: 75298.6 | physical tokens/s: 76098.3 | loss tokens/s: 57996.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.387532E-01 | lm-global-supervised-tokens: 1.292209E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.700995E-02 | lm-token-buffer-mib: 2.347440E+02 | global_load_balancing_loss: 9.999941E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11214.117 | params rms: 0.060721 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:31:30.425318 | finish at 2026-10-17 17:44:39 + [2026-10-12 13:16:52] iteration 373/ 2384 | consumed samples: 11936 | elapsed time per iteration (ms): 223795.4 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.61% | FLOPs estimator: workload-aware | logical tokens/s: 74966.6 | valid tokens/s: 74966.6 | physical tokens/s: 75716.3 | loss tokens/s: 56449.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.966233E-01 | lm-global-supervised-tokens: 1.263315E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.682421E-02 | lm-token-buffer-mib: 2.341541E+02 | global_load_balancing_loss: 1.000800E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11214.060 | params rms: 0.060720 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:00:52.492152 | finish at 2026-10-17 18:17:45 + [2026-10-12 13:20:35] iteration 374/ 2384 | consumed samples: 11968 | elapsed time per iteration (ms): 223129.0 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75190.4 | valid tokens/s: 75190.4 | physical tokens/s: 76005.5 | loss tokens/s: 54328.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.360067E-01 | lm-global-supervised-tokens: 1.212235E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.516665E-02 | lm-token-buffer-mib: 2.325510E+02 | global_load_balancing_loss: 9.999342E-01 | loss scale: 1.0 | grad norm: 0.273 | params norm: 11213.994 | params rms: 0.060720 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:34:49.270728 | finish at 2026-10-17 17:55:24 + [2026-10-12 13:24:23] iteration 375/ 2384 | consumed samples: 12000 | elapsed time per iteration (ms): 228050.8 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 73567.7 | valid tokens/s: 73567.7 | physical tokens/s: 74465.9 | loss tokens/s: 56328.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.438233E-01 | lm-global-supervised-tokens: 1.284585E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 7.931596E-02 | lm-token-buffer-mib: 2.356559E+02 | global_load_balancing_loss: 1.000857E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11213.931 | params rms: 0.060720 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:15:54.082757 | finish at 2026-10-17 20:40:17 + [2026-10-12 13:28:07] iteration 376/ 2384 | consumed samples: 12032 | elapsed time per iteration (ms): 224172.1 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 74840.8 | valid tokens/s: 74840.8 | physical tokens/s: 75503.1 | loss tokens/s: 53899.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.455889E-01 | lm-global-supervised-tokens: 1.208271E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.372001E-02 | lm-token-buffer-mib: 2.349600E+02 | global_load_balancing_loss: 9.997230E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11213.872 | params rms: 0.060719 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:02:17.586510 | finish at 2026-10-17 18:30:25 + [2026-10-12 13:31:48] iteration 377/ 2384 | consumed samples: 12064 | elapsed time per iteration (ms): 220517.0 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 76081.1 | valid tokens/s: 76081.1 | physical tokens/s: 76568.6 | loss tokens/s: 55055.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.071288E-01 | lm-global-supervised-tokens: 1.214061E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.292663E-02 | lm-token-buffer-mib: 2.330813E+02 | global_load_balancing_loss: 1.000383E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11213.817 | params rms: 0.060719 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:56:17.666770 | finish at 2026-10-17 16:28:06 + [2026-10-12 13:35:29] iteration 378/ 2384 | consumed samples: 12096 | elapsed time per iteration (ms): 221334.0 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75800.3 | valid tokens/s: 75800.3 | physical tokens/s: 76720.8 | loss tokens/s: 56822.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.653598E-01 | lm-global-supervised-tokens: 1.257678E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.184181E-02 | lm-token-buffer-mib: 2.328899E+02 | global_load_balancing_loss: 1.000773E+00 | loss scale: 1.0 | grad norm: 0.269 | params norm: 11213.760 | params rms: 0.060719 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:19:56.015222 | finish at 2026-10-17 16:55:25 + [2026-10-12 13:39:09] iteration 379/ 2384 | consumed samples: 12128 | elapsed time per iteration (ms): 219614.3 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 76393.9 | valid tokens/s: 76393.9 | physical tokens/s: 77022.0 | loss tokens/s: 51608.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.982088E-01 | lm-global-supervised-tokens: 1.133392E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.325328E-02 | lm-token-buffer-mib: 2.329887E+02 | global_load_balancing_loss: 1.001464E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11213.702 | params rms: 0.060719 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:18:46.735581 | finish at 2026-10-17 15:57:56 + [2026-10-12 13:43:55] iteration 380/ 2384 | consumed samples: 12160 | elapsed time per iteration (ms): 286639.4 | throughput per GPU (TFLOP/s/GPU): 18.0 | MFU 5.76% | FLOPs estimator: workload-aware | logical tokens/s: 58530.6 | valid tokens/s: 58530.6 | physical tokens/s: 59995.2 | loss tokens/s: 46225.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.733801E-01 | lm-global-supervised-tokens: 1.325008E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.508665E-02 | lm-token-buffer-mib: 2.336989E+02 | global_load_balancing_loss: 1.001729E+00 | loss scale: 1.0 | grad norm: 0.199 | params norm: 11213.645 | params rms: 0.060718 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:33:45.378633 | finish at 2026-10-19 05:17:41 + [2026-10-12 13:47:41] iteration 381/ 2384 | consumed samples: 12192 | elapsed time per iteration (ms): 225519.5 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.68% | FLOPs estimator: workload-aware | logical tokens/s: 74393.5 | valid tokens/s: 74393.5 | physical tokens/s: 75190.0 | loss tokens/s: 56328.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.811887E-01 | lm-global-supervised-tokens: 1.270309E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.678250E-02 | lm-token-buffer-mib: 2.324070E+02 | global_load_balancing_loss: 1.001296E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11213.588 | params rms: 0.060718 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:28:35.462069 | finish at 2026-10-17 19:16:16 + [2026-10-12 13:51:23] iteration 382/ 2384 | consumed samples: 12224 | elapsed time per iteration (ms): 222404.8 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 75435.4 | valid tokens/s: 75435.4 | physical tokens/s: 76153.8 | loss tokens/s: 57472.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.406943E-01 | lm-global-supervised-tokens: 1.278225E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.727202E-02 | lm-token-buffer-mib: 2.328143E+02 | global_load_balancing_loss: 9.996349E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11213.532 | params rms: 0.060718 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:40:54.412817 | finish at 2026-10-17 17:32:18 + [2026-10-12 13:55:03] iteration 383/ 2384 | consumed samples: 12256 | elapsed time per iteration (ms): 219903.7 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 76293.3 | valid tokens/s: 76293.3 | physical tokens/s: 77005.9 | loss tokens/s: 59101.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.945024E-01 | lm-global-supervised-tokens: 1.299660E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.428097E-02 | lm-token-buffer-mib: 2.340922E+02 | global_load_balancing_loss: 9.996247E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11213.477 | params rms: 0.060717 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:13:47.280073 | finish at 2026-10-17 16:08:51 + [2026-10-12 13:58:49] iteration 384/ 2384 | consumed samples: 12288 | elapsed time per iteration (ms): 225598.0 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 74367.7 | valid tokens/s: 74367.7 | physical tokens/s: 75219.1 | loss tokens/s: 57477.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.130793E-01 | lm-global-supervised-tokens: 1.296673E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.575893E-02 | lm-token-buffer-mib: 2.358247E+02 | global_load_balancing_loss: 1.002483E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11213.421 | params rms: 0.060717 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:19:56.098804 | finish at 2026-10-17 19:18:45 + [2026-10-12 14:02:30] iteration 385/ 2384 | consumed samples: 12320 | elapsed time per iteration (ms): 221281.4 | throughput per GPU (TFLOP/s/GPU): 27.1 | MFU 8.68% | FLOPs estimator: workload-aware | logical tokens/s: 75818.3 | valid tokens/s: 75818.3 | physical tokens/s: 75992.9 | loss tokens/s: 55068.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.151815E-01 | lm-global-supervised-tokens: 1.218573E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.328522E-02 | lm-token-buffer-mib: 2.328018E+02 | global_load_balancing_loss: 1.002147E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11213.365 | params rms: 0.060717 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:52:21.495177 | finish at 2026-10-17 16:54:52 + [2026-10-12 14:06:16] iteration 386/ 2384 | consumed samples: 12352 | elapsed time per iteration (ms): 225468.7 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 74410.3 | valid tokens/s: 74410.3 | physical tokens/s: 75010.5 | loss tokens/s: 53122.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.754974E-01 | lm-global-supervised-tokens: 1.197740E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.330977E-02 | lm-token-buffer-mib: 2.325142E+02 | global_load_balancing_loss: 9.996600E-01 | loss scale: 1.0 | grad norm: 0.182 | params norm: 11213.304 | params rms: 0.060716 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:08:06.531060 | finish at 2026-10-17 19:14:22 + [2026-10-12 14:09:55] iteration 387/ 2384 | consumed samples: 12384 | elapsed time per iteration (ms): 219574.2 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 76407.9 | valid tokens/s: 76407.9 | physical tokens/s: 76681.9 | loss tokens/s: 56619.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.153548E-01 | lm-global-supervised-tokens: 1.243215E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.021326E-01 | lm-token-buffer-mib: 2.327575E+02 | global_load_balancing_loss: 9.988811E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11213.245 | params rms: 0.060716 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:48:09.731984 | finish at 2026-10-17 15:58:05 + [2026-10-12 14:13:37] iteration 388/ 2384 | consumed samples: 12416 | elapsed time per iteration (ms): 221366.5 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75789.3 | valid tokens/s: 75789.3 | physical tokens/s: 76256.8 | loss tokens/s: 57706.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.768140E-01 | lm-global-supervised-tokens: 1.277434E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.195135E-02 | lm-token-buffer-mib: 2.338704E+02 | global_load_balancing_loss: 1.000308E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11213.188 | params rms: 0.060716 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:44:07.507153 | finish at 2026-10-17 16:57:44 + [2026-10-12 14:17:22] iteration 389/ 2384 | consumed samples: 12448 | elapsed time per iteration (ms): 224937.5 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 74586.1 | valid tokens/s: 74586.0 | physical tokens/s: 75266.1 | loss tokens/s: 55960.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.487544E-01 | lm-global-supervised-tokens: 1.258754E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.092300E-02 | lm-token-buffer-mib: 2.349504E+02 | global_load_balancing_loss: 9.997199E-01 | loss scale: 1.0 | grad norm: 0.327 | params norm: 11213.133 | params rms: 0.060715 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:39:10.375761 | finish at 2026-10-17 18:56:32 + [2026-10-12 14:21:27] iteration 390/ 2384 | consumed samples: 12480 | elapsed time per iteration (ms): 245532.7 | throughput per GPU (TFLOP/s/GPU): 18.4 | MFU 5.89% | FLOPs estimator: workload-aware | logical tokens/s: 68329.8 | valid tokens/s: 68329.8 | physical tokens/s: 69269.3 | loss tokens/s: 53050.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.731595E-01 | lm-global-supervised-tokens: 1.302574E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.176073E-02 | lm-token-buffer-mib: 2.355426E+02 | global_load_balancing_loss: 1.001675E+00 | loss scale: 1.0 | grad norm: 0.475 | params norm: 11213.082 | params rms: 0.060715 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:59:52.180628 | finish at 2026-10-18 06:21:19 + [2026-10-12 14:25:10] iteration 391/ 2384 | consumed samples: 12512 | elapsed time per iteration (ms): 222421.9 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.23% | FLOPs estimator: workload-aware | logical tokens/s: 75429.6 | valid tokens/s: 75429.6 | physical tokens/s: 76515.1 | loss tokens/s: 59699.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.357657E-01 | lm-global-supervised-tokens: 1.327840E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.728347E-02 | lm-token-buffer-mib: 2.351317E+02 | global_load_balancing_loss: 1.000521E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11213.031 | params rms: 0.060715 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:08:06.751259 | finish at 2026-10-17 17:33:16 + [2026-10-12 14:28:50] iteration 392/ 2384 | consumed samples: 12544 | elapsed time per iteration (ms): 220689.6 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 76021.7 | valid tokens/s: 76021.7 | physical tokens/s: 76837.5 | loss tokens/s: 55411.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.662661E-01 | lm-global-supervised-tokens: 1.222867E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.378941E-02 | lm-token-buffer-mib: 2.339968E+02 | global_load_balancing_loss: 1.000957E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11212.982 | params rms: 0.060715 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:06:53.686981 | finish at 2026-10-17 16:35:44 + [2026-10-12 14:32:34] iteration 393/ 2384 | consumed samples: 12576 | elapsed time per iteration (ms): 223622.6 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.25% | FLOPs estimator: workload-aware | logical tokens/s: 75024.6 | valid tokens/s: 75024.6 | physical tokens/s: 75241.1 | loss tokens/s: 56383.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.347969E-01 | lm-global-supervised-tokens: 1.260853E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.216119E-02 | lm-token-buffer-mib: 2.324520E+02 | global_load_balancing_loss: 1.000492E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11212.934 | params rms: 0.060714 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:40:32.584414 | finish at 2026-10-17 18:13:06 + [2026-10-12 14:36:15] iteration 394/ 2384 | consumed samples: 12608 | elapsed time per iteration (ms): 220771.9 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75993.4 | valid tokens/s: 75993.4 | physical tokens/s: 76477.0 | loss tokens/s: 56615.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.675877E-01 | lm-global-supervised-tokens: 1.249902E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.544890E-02 | lm-token-buffer-mib: 2.333966E+02 | global_load_balancing_loss: 1.002244E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11212.883 | params rms: 0.060714 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:02:16.072338 | finish at 2026-10-17 16:38:31 + [2026-10-12 14:40:00] iteration 395/ 2384 | consumed samples: 12640 | elapsed time per iteration (ms): 225609.9 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.60% | FLOPs estimator: workload-aware | logical tokens/s: 74363.7 | valid tokens/s: 74363.7 | physical tokens/s: 75183.1 | loss tokens/s: 55347.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.346245E-01 | lm-global-supervised-tokens: 1.248692E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.550997E-02 | lm-token-buffer-mib: 2.340531E+02 | global_load_balancing_loss: 1.002088E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11212.831 | params rms: 0.060714 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:38:58.189258 | finish at 2026-10-17 19:18:58 + [2026-10-12 14:43:48] iteration 396/ 2384 | consumed samples: 12672 | elapsed time per iteration (ms): 227514.9 | throughput per GPU (TFLOP/s/GPU): 18.0 | MFU 5.78% | FLOPs estimator: workload-aware | logical tokens/s: 73741.1 | valid tokens/s: 73741.1 | physical tokens/s: 74862.4 | loss tokens/s: 55297.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.890446E-01 | lm-global-supervised-tokens: 1.258104E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.682683E-02 | lm-token-buffer-mib: 2.377724E+02 | global_load_balancing_loss: 1.001441E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11212.779 | params rms: 0.060714 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:38:19.693183 | finish at 2026-10-17 20:22:07 + [2026-10-12 14:47:31] iteration 397/ 2384 | consumed samples: 12704 | elapsed time per iteration (ms): 223533.6 | throughput per GPU (TFLOP/s/GPU): 26.6 | MFU 8.53% | FLOPs estimator: workload-aware | logical tokens/s: 75054.5 | valid tokens/s: 75054.5 | physical tokens/s: 75760.3 | loss tokens/s: 60566.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.978392E-01 | lm-global-supervised-tokens: 1.353856E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.217544E-02 | lm-token-buffer-mib: 2.348644E+02 | global_load_balancing_loss: 1.000650E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11212.728 | params rms: 0.060713 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:22:41.170530 | finish at 2026-10-17 18:10:12 + [2026-10-12 14:51:14] iteration 398/ 2384 | consumed samples: 12736 | elapsed time per iteration (ms): 222360.8 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 75450.3 | valid tokens/s: 75450.3 | physical tokens/s: 76136.3 | loss tokens/s: 57011.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.037004E-01 | lm-global-supervised-tokens: 1.267720E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.892513E-02 | lm-token-buffer-mib: 2.333229E+02 | global_load_balancing_loss: 9.996143E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11212.678 | params rms: 0.060713 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:40:08.555484 | finish at 2026-10-17 17:31:22 + [2026-10-12 14:54:57] iteration 399/ 2384 | consumed samples: 12768 | elapsed time per iteration (ms): 223121.4 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.60% | FLOPs estimator: workload-aware | logical tokens/s: 75193.2 | valid tokens/s: 75193.2 | physical tokens/s: 75952.5 | loss tokens/s: 58620.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.689251E-01 | lm-global-supervised-tokens: 1.307945E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.710530E-02 | lm-token-buffer-mib: 2.351126E+02 | global_load_balancing_loss: 1.000486E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11212.625 | params rms: 0.060713 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:01:35.953678 | finish at 2026-10-17 17:56:33 + [2026-10-12 14:58:59] iteration 400/ 2384 | consumed samples: 12800 | elapsed time per iteration (ms): 241778.6 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 69390.8 | valid tokens/s: 69390.8 | physical tokens/s: 69858.0 | loss tokens/s: 50908.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.680460E-01 | lm-global-supervised-tokens: 1.230868E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.372385E-02 | lm-token-buffer-mib: 2.332155E+02 | global_load_balancing_loss: 9.991992E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11212.573 | params rms: 0.060712 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:14:48.657211 | finish at 2026-10-18 04:13:47 +times across ranks (ms): + save-checkpoint: + rank 0: 25593.10 + rank 1: 25593.10 + rank 2: 25593.12 + rank 3: 25593.11 + rank 4: 25593.11 + rank 5: 25593.10 + rank 6: 25592.99 + rank 7: 25593.12 + rank 8: 25593.03 + rank 9: 25593.05 + rank 10: 25593.06 + rank 11: 25593.04 + rank 12: 25593.04 + rank 13: 25593.07 + rank 14: 25593.05 + rank 15: 25593.07 + rank 16: 25592.91 + rank 17: 25592.92 + rank 18: 25592.93 + rank 19: 25592.92 + rank 20: 25592.90 + rank 21: 25592.89 + rank 22: 25592.91 + rank 23: 25592.90 + rank 24: 25592.81 + rank 25: 25592.68 + rank 26: 25593.47 + rank 27: 25592.78 + rank 28: 25592.73 + rank 29: 25592.79 + rank 30: 25592.75 + rank 31: 25592.73 + rank 32: 25593.32 + rank 33: 25593.30 + rank 34: 25593.31 + rank 35: 25593.34 + rank 36: 25593.25 + rank 37: 25593.31 + rank 38: 25593.26 + rank 39: 25593.24 + rank 40: 25593.27 + rank 41: 25593.26 + rank 42: 25593.26 + rank 43: 25593.27 + rank 44: 25593.24 + rank 45: 25593.26 + rank 46: 25593.24 + rank 47: 25593.26 + rank 48: 25593.49 + rank 49: 25593.46 + rank 50: 25593.41 + rank 51: 25593.45 + rank 52: 25593.46 + rank 53: 25593.39 + rank 54: 25593.39 + rank 55: 25593.44 + rank 56: 25592.73 + rank 57: 25592.56 + rank 58: 25592.76 + rank 59: 25592.75 + rank 60: 25592.77 + rank 61: 25592.71 + rank 62: 25592.71 + rank 63: 25592.75 + [2026-10-12 15:03:08] iteration 401/ 2384 | consumed samples: 12832 | elapsed time per iteration (ms): 224154.1 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 74846.7 | valid tokens/s: 74846.7 | physical tokens/s: 75360.2 | loss tokens/s: 54920.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.456723E-01 | lm-global-supervised-tokens: 1.231067E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 9.157430E-02 | lm-token-buffer-mib: 2.338413E+02 | global_load_balancing_loss: 1.002198E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11212.520 | params rms: 0.060712 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:28:17.580184 | finish at 2026-10-17 18:31:26 + [2026-10-12 15:06:54] iteration 402/ 2384 | consumed samples: 12864 | elapsed time per iteration (ms): 225555.4 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 74381.6 | valid tokens/s: 74381.6 | physical tokens/s: 75035.0 | loss tokens/s: 58210.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.443243E-01 | lm-global-supervised-tokens: 1.312977E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.340086E-02 | lm-token-buffer-mib: 2.327129E+02 | global_load_balancing_loss: 9.993671E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11212.470 | params rms: 0.060712 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:10:50.897100 | finish at 2026-10-17 19:17:45 + [2026-10-12 15:10:35] iteration 403/ 2384 | consumed samples: 12896 | elapsed time per iteration (ms): 221264.9 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.80% | FLOPs estimator: workload-aware | logical tokens/s: 75824.0 | valid tokens/s: 75824.0 | physical tokens/s: 76243.5 | loss tokens/s: 60552.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.184148E-01 | lm-global-supervised-tokens: 1.339813E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.825737E-02 | lm-token-buffer-mib: 2.335921E+02 | global_load_balancing_loss: 9.993334E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11212.418 | params rms: 0.060712 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:45:25.838629 | finish at 2026-10-17 16:56:01 + [2026-10-12 15:14:19] iteration 404/ 2384 | consumed samples: 12928 | elapsed time per iteration (ms): 223381.8 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 75105.4 | valid tokens/s: 75105.4 | physical tokens/s: 76336.7 | loss tokens/s: 58038.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.484210E-01 | lm-global-supervised-tokens: 1.296479E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.439536E-02 | lm-token-buffer-mib: 2.370970E+02 | global_load_balancing_loss: 9.996281E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11212.371 | params rms: 0.060711 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:51:35.956793 | finish at 2026-10-17 18:05:54 + [2026-10-12 15:18:00] iteration 405/ 2384 | consumed samples: 12960 | elapsed time per iteration (ms): 221662.7 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.90% | FLOPs estimator: workload-aware | logical tokens/s: 75687.9 | valid tokens/s: 75687.9 | physical tokens/s: 76254.8 | loss tokens/s: 60185.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.472794E-01 | lm-global-supervised-tokens: 1.334085E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.164460E-02 | lm-token-buffer-mib: 2.326915E+02 | global_load_balancing_loss: 1.001909E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11212.326 | params rms: 0.060711 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:51:10.518092 | finish at 2026-10-17 17:09:11 + [2026-10-12 15:21:46] iteration 406/ 2384 | consumed samples: 12992 | elapsed time per iteration (ms): 226120.5 | throughput per GPU (TFLOP/s/GPU): 26.9 | MFU 8.62% | FLOPs estimator: workload-aware | logical tokens/s: 74195.5 | valid tokens/s: 74195.5 | physical tokens/s: 74753.8 | loss tokens/s: 56310.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.360933E-01 | lm-global-supervised-tokens: 1.273304E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.724912E-02 | lm-token-buffer-mib: 2.340874E+02 | global_load_balancing_loss: 1.001128E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11212.281 | params rms: 0.060711 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:14:26.352475 | finish at 2026-10-17 19:36:13 + [2026-10-12 15:25:30] iteration 407/ 2384 | consumed samples: 13024 | elapsed time per iteration (ms): 223951.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 74914.6 | valid tokens/s: 74914.6 | physical tokens/s: 75609.6 | loss tokens/s: 56192.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.425485E-01 | lm-global-supervised-tokens: 1.258437E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.345489E-02 | lm-token-buffer-mib: 2.339507E+02 | global_load_balancing_loss: 1.002100E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11212.237 | params rms: 0.060711 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:59:11.424846 | finish at 2026-10-17 18:24:42 + [2026-10-12 15:29:13] iteration 408/ 2384 | consumed samples: 13056 | elapsed time per iteration (ms): 222684.7 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.88% | FLOPs estimator: workload-aware | logical tokens/s: 75340.6 | valid tokens/s: 75340.6 | physical tokens/s: 76055.2 | loss tokens/s: 54979.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.165957E-01 | lm-global-supervised-tokens: 1.224299E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.328037E-02 | lm-token-buffer-mib: 2.336228E+02 | global_load_balancing_loss: 1.000517E+00 | loss scale: 1.0 | grad norm: 0.223 | params norm: 11212.192 | params rms: 0.060710 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:13:44.954504 | finish at 2026-10-17 17:42:58 + [2026-10-12 15:32:55] iteration 409/ 2384 | consumed samples: 13088 | elapsed time per iteration (ms): 222153.5 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 75520.8 | valid tokens/s: 75520.8 | physical tokens/s: 76636.0 | loss tokens/s: 54207.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.632878E-01 | lm-global-supervised-tokens: 1.204236E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.999228E-02 | lm-token-buffer-mib: 2.343073E+02 | global_load_balancing_loss: 1.000054E+00 | loss scale: 1.0 | grad norm: 0.268 | params norm: 11212.146 | params rms: 0.060710 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:52:33.065187 | finish at 2026-10-17 17:25:28 + [2026-10-12 15:36:59] iteration 410/ 2384 | consumed samples: 13120 | elapsed time per iteration (ms): 243881.9 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.65% | FLOPs estimator: workload-aware | logical tokens/s: 68792.3 | valid tokens/s: 68792.3 | physical tokens/s: 69294.4 | loss tokens/s: 53567.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 9.176591E-01 | lm-global-supervised-tokens: 1.306402E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.492695E-02 | lm-token-buffer-mib: 2.334133E+02 | global_load_balancing_loss: 9.990575E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11212.101 | params rms: 0.060710 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:43:42.839680 | finish at 2026-10-18 05:20:42 + [2026-10-12 15:40:41] iteration 411/ 2384 | consumed samples: 13152 | elapsed time per iteration (ms): 221988.5 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.75% | FLOPs estimator: workload-aware | logical tokens/s: 75576.7 | valid tokens/s: 75576.7 | physical tokens/s: 76271.8 | loss tokens/s: 58680.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.863357E-01 | lm-global-supervised-tokens: 1.302637E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.459828E-02 | lm-token-buffer-mib: 2.342469E+02 | global_load_balancing_loss: 9.998680E-01 | loss scale: 1.0 | grad norm: 0.253 | params norm: 11212.056 | params rms: 0.060710 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:39:43.362948 | finish at 2026-10-17 17:20:24 + [2026-10-12 15:44:29] iteration 412/ 2384 | consumed samples: 13184 | elapsed time per iteration (ms): 227720.2 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.27% | FLOPs estimator: workload-aware | logical tokens/s: 73674.6 | valid tokens/s: 73674.6 | physical tokens/s: 73981.3 | loss tokens/s: 55598.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.931722E-01 | lm-global-supervised-tokens: 1.266087E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.241493E-02 | lm-token-buffer-mib: 2.329836E+02 | global_load_balancing_loss: 1.002981E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11212.007 | params rms: 0.060709 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:44:24.197849 | finish at 2026-10-17 20:28:53 + [2026-10-12 15:48:15] iteration 413/ 2384 | consumed samples: 13216 | elapsed time per iteration (ms): 226079.4 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 74209.3 | valid tokens/s: 74209.3 | physical tokens/s: 75325.6 | loss tokens/s: 51729.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.611777E-01 | lm-global-supervised-tokens: 1.169504E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.355954E-02 | lm-token-buffer-mib: 2.358356E+02 | global_load_balancing_loss: 1.001764E+00 | loss scale: 1.0 | grad norm: 0.397 | params norm: 11211.953 | params rms: 0.060709 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:46:42.495174 | finish at 2026-10-17 19:34:57 + [2026-10-12 15:52:01] iteration 414/ 2384 | consumed samples: 13248 | elapsed time per iteration (ms): 226078.6 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 74209.5 | valid tokens/s: 74209.5 | physical tokens/s: 75079.3 | loss tokens/s: 58002.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.324914E-01 | lm-global-supervised-tokens: 1.311311E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.651768E-02 | lm-token-buffer-mib: 2.339889E+02 | global_load_balancing_loss: 9.998171E-01 | loss scale: 1.0 | grad norm: 0.204 | params norm: 11211.900 | params rms: 0.060709 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:42:54.831059 | finish at 2026-10-17 19:34:56 + [2026-10-12 15:55:44] iteration 415/ 2384 | consumed samples: 13280 | elapsed time per iteration (ms): 223604.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75030.8 | valid tokens/s: 75030.8 | physical tokens/s: 75549.8 | loss tokens/s: 62400.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.624650E-01 | lm-global-supervised-tokens: 1.395308E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.246196E-02 | lm-token-buffer-mib: 2.331272E+02 | global_load_balancing_loss: 9.989058E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11211.854 | params rms: 0.060709 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:17:56.425464 | finish at 2026-10-17 18:13:41 + [2026-10-12 15:59:31] iteration 416/ 2384 | consumed samples: 13312 | elapsed time per iteration (ms): 226498.9 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 74071.9 | valid tokens/s: 74071.9 | physical tokens/s: 74509.3 | loss tokens/s: 53251.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.078928E-01 | lm-global-supervised-tokens: 1.206142E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.409408E-02 | lm-token-buffer-mib: 2.337095E+02 | global_load_balancing_loss: 9.990239E-01 | loss scale: 1.0 | grad norm: 0.206 | params norm: 11211.808 | params rms: 0.060708 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:49:09.922817 | finish at 2026-10-17 19:48:41 + [2026-10-12 16:03:12] iteration 417/ 2384 | consumed samples: 13344 | elapsed time per iteration (ms): 221102.6 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.48% | FLOPs estimator: workload-aware | logical tokens/s: 75879.7 | valid tokens/s: 75879.7 | physical tokens/s: 76394.4 | loss tokens/s: 56239.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.852913E-01 | lm-global-supervised-tokens: 1.243463E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.246354E-02 | lm-token-buffer-mib: 2.328845E+02 | global_load_balancing_loss: 9.999898E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11211.757 | params rms: 0.060708 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:48:28.819082 | finish at 2026-10-17 16:51:41 + [2026-10-12 16:06:55] iteration 418/ 2384 | consumed samples: 13376 | elapsed time per iteration (ms): 222816.7 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 75295.8 | valid tokens/s: 75295.8 | physical tokens/s: 76141.6 | loss tokens/s: 59534.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.318370E-01 | lm-global-supervised-tokens: 1.326534E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.787097E-02 | lm-token-buffer-mib: 2.337239E+02 | global_load_balancing_loss: 1.001277E+00 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11211.707 | params rms: 0.060708 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:40:57.708567 | finish at 2026-10-17 17:47:53 + [2026-10-12 16:10:38] iteration 419/ 2384 | consumed samples: 13408 | elapsed time per iteration (ms): 223227.5 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 75157.4 | valid tokens/s: 75157.4 | physical tokens/s: 76091.3 | loss tokens/s: 57599.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.072636E-01 | lm-global-supervised-tokens: 1.285773E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.765816E-02 | lm-token-buffer-mib: 2.360801E+02 | global_load_balancing_loss: 1.002158E+00 | loss scale: 1.0 | grad norm: 0.226 | params norm: 11211.662 | params rms: 0.060708 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:50:41.949817 | finish at 2026-10-17 18:01:20 + [2026-10-12 16:14:48] iteration 420/ 2384 | consumed samples: 13440 | elapsed time per iteration (ms): 249902.8 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 67134.9 | valid tokens/s: 67134.9 | physical tokens/s: 68145.3 | loss tokens/s: 53420.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.386282E-01 | lm-global-supervised-tokens: 1.334988E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.319440E-02 | lm-token-buffer-mib: 2.358864E+02 | global_load_balancing_loss: 1.001312E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11211.619 | params rms: 0.060707 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:20:09.182713 | finish at 2026-10-18 08:34:57 + [2026-10-12 16:18:31] iteration 421/ 2384 | consumed samples: 13472 | elapsed time per iteration (ms): 222626.6 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 75360.2 | valid tokens/s: 75360.2 | physical tokens/s: 76254.7 | loss tokens/s: 56554.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.295505E-01 | lm-global-supervised-tokens: 1.259062E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.811438E-02 | lm-token-buffer-mib: 2.344466E+02 | global_load_balancing_loss: 1.002131E+00 | loss scale: 1.0 | grad norm: 0.402 | params norm: 11211.576 | params rms: 0.060707 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:23:35.946587 | finish at 2026-10-17 17:42:07 + [2026-10-12 16:22:11] iteration 422/ 2384 | consumed samples: 13504 | elapsed time per iteration (ms): 220258.9 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 76170.4 | valid tokens/s: 76170.4 | physical tokens/s: 77141.8 | loss tokens/s: 58712.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.154279E-01 | lm-global-supervised-tokens: 1.293197E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.343947E-02 | lm-token-buffer-mib: 2.353303E+02 | global_load_balancing_loss: 1.000497E+00 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11211.537 | params rms: 0.060707 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:02:27.911605 | finish at 2026-10-17 16:24:39 + [2026-10-12 16:25:53] iteration 423/ 2384 | consumed samples: 13536 | elapsed time per iteration (ms): 222209.0 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 75501.9 | valid tokens/s: 75501.9 | physical tokens/s: 76071.9 | loss tokens/s: 60563.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.639549E-01 | lm-global-supervised-tokens: 1.345774E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.527655E-02 | lm-token-buffer-mib: 2.336232E+02 | global_load_balancing_loss: 1.001105E+00 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11211.495 | params rms: 0.060707 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:02:31.877737 | finish at 2026-10-17 17:28:25 + [2026-10-12 16:29:39] iteration 424/ 2384 | consumed samples: 13568 | elapsed time per iteration (ms): 226318.6 | throughput per GPU (TFLOP/s/GPU): 28.8 | MFU 9.24% | FLOPs estimator: workload-aware | logical tokens/s: 74130.7 | valid tokens/s: 74130.7 | physical tokens/s: 74530.8 | loss tokens/s: 58007.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.843873E-01 | lm-global-supervised-tokens: 1.312821E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.568324E-02 | lm-token-buffer-mib: 2.338009E+02 | global_load_balancing_loss: 1.001598E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11211.450 | params rms: 0.060706 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:13:04.489527 | finish at 2026-10-17 19:42:44 + [2026-10-12 16:33:21] iteration 425/ 2384 | consumed samples: 13600 | elapsed time per iteration (ms): 221542.2 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75729.1 | valid tokens/s: 75729.1 | physical tokens/s: 76217.4 | loss tokens/s: 56792.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.326539E-01 | lm-global-supervised-tokens: 1.258186E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.694295E-02 | lm-token-buffer-mib: 2.344137E+02 | global_load_balancing_loss: 9.996480E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11211.406 | params rms: 0.060706 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:33:21.197530 | finish at 2026-10-17 17:06:42 + [2026-10-12 16:37:09] iteration 426/ 2384 | consumed samples: 13632 | elapsed time per iteration (ms): 227975.0 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 73592.3 | valid tokens/s: 73592.3 | physical tokens/s: 74738.3 | loss tokens/s: 59278.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.604973E-01 | lm-global-supervised-tokens: 1.351411E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.374462E-02 | lm-token-buffer-mib: 2.326497E+02 | global_load_balancing_loss: 9.988051E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11211.359 | params rms: 0.060706 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:59:34.974655 | finish at 2026-10-17 20:36:44 + [2026-10-12 16:40:54] iteration 427/ 2384 | consumed samples: 13664 | elapsed time per iteration (ms): 224886.0 | throughput per GPU (TFLOP/s/GPU): 28.6 | MFU 9.16% | FLOPs estimator: workload-aware | logical tokens/s: 74603.0 | valid tokens/s: 74603.0 | physical tokens/s: 74797.6 | loss tokens/s: 57822.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.363905E-01 | lm-global-supervised-tokens: 1.300345E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.688281E-02 | lm-token-buffer-mib: 2.324808E+02 | global_load_balancing_loss: 9.992049E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11211.315 | params rms: 0.060706 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:15:01.944765 | finish at 2026-10-17 18:55:56 + [2026-10-12 16:44:41] iteration 428/ 2384 | consumed samples: 13696 | elapsed time per iteration (ms): 227393.8 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 73780.3 | valid tokens/s: 73780.3 | physical tokens/s: 74392.3 | loss tokens/s: 51530.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.929936E-01 | lm-global-supervised-tokens: 1.171768E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.871765E-02 | lm-token-buffer-mib: 2.331430E+02 | global_load_balancing_loss: 1.001840E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11211.271 | params rms: 0.060705 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:33:02.346056 | finish at 2026-10-17 20:17:44 + [2026-10-12 16:48:30] iteration 429/ 2384 | consumed samples: 13728 | elapsed time per iteration (ms): 228626.9 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 73382.3 | valid tokens/s: 73382.3 | physical tokens/s: 74423.6 | loss tokens/s: 53420.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.403390E-01 | lm-global-supervised-tokens: 1.221330E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.454162E-02 | lm-token-buffer-mib: 2.380824E+02 | global_load_balancing_loss: 1.000167E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11211.225 | params rms: 0.060705 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:09:25.661664 | finish at 2026-10-17 20:57:56 + [2026-10-12 16:52:34] iteration 430/ 2384 | consumed samples: 13760 | elapsed time per iteration (ms): 243929.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 68778.8 | valid tokens/s: 68778.8 | physical tokens/s: 69214.9 | loss tokens/s: 51752.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.268270E-01 | lm-global-supervised-tokens: 1.262399E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.392841E-02 | lm-token-buffer-mib: 2.337599E+02 | global_load_balancing_loss: 1.002479E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11211.177 | params rms: 0.060705 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:23:57.398113 | finish at 2026-10-18 05:16:31 + [2026-10-12 16:56:17] iteration 431/ 2384 | consumed samples: 13792 | elapsed time per iteration (ms): 223588.6 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.76% | FLOPs estimator: workload-aware | logical tokens/s: 75036.0 | valid tokens/s: 75036.0 | physical tokens/s: 75412.2 | loss tokens/s: 55748.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.835559E-01 | lm-global-supervised-tokens: 1.246482E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.859441E-02 | lm-token-buffer-mib: 2.331151E+02 | global_load_balancing_loss: 1.001023E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11211.127 | params rms: 0.060705 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:17:48.537041 | finish at 2026-10-17 18:14:06 + [2026-10-12 17:00:00] iteration 432/ 2384 | consumed samples: 13824 | elapsed time per iteration (ms): 222569.1 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 75379.8 | valid tokens/s: 75379.8 | physical tokens/s: 76109.9 | loss tokens/s: 56529.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.737849E-01 | lm-global-supervised-tokens: 1.258178E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.462971E-02 | lm-token-buffer-mib: 2.345482E+02 | global_load_balancing_loss: 9.993416E-01 | loss scale: 1.0 | grad norm: 0.193 | params norm: 11211.078 | params rms: 0.060704 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:40:54.808548 | finish at 2026-10-17 17:40:55 + [2026-10-12 17:03:42] iteration 433/ 2384 | consumed samples: 13856 | elapsed time per iteration (ms): 221885.7 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.22% | FLOPs estimator: workload-aware | logical tokens/s: 75611.7 | valid tokens/s: 75611.7 | physical tokens/s: 76694.5 | loss tokens/s: 60126.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.690501E-01 | lm-global-supervised-tokens: 1.334128E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.154776E-02 | lm-token-buffer-mib: 2.340989E+02 | global_load_balancing_loss: 1.001116E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11211.022 | params rms: 0.060704 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:14:59.029980 | finish at 2026-10-17 17:18:41 + [2026-10-12 17:07:21] iteration 434/ 2384 | consumed samples: 13888 | elapsed time per iteration (ms): 219623.1 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.49% | FLOPs estimator: workload-aware | logical tokens/s: 76390.8 | valid tokens/s: 76390.8 | physical tokens/s: 76788.4 | loss tokens/s: 57250.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.045730E-01 | lm-global-supervised-tokens: 1.257349E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.657829E-02 | lm-token-buffer-mib: 2.340616E+02 | global_load_balancing_loss: 1.000298E+00 | loss scale: 1.0 | grad norm: 0.262 | params norm: 11210.969 | params rms: 0.060704 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:57:44.981389 | finish at 2026-10-17 16:05:06 + [2026-10-12 17:11:04] iteration 435/ 2384 | consumed samples: 13920 | elapsed time per iteration (ms): 222994.9 | throughput per GPU (TFLOP/s/GPU): 26.4 | MFU 8.46% | FLOPs estimator: workload-aware | logical tokens/s: 75235.6 | valid tokens/s: 75235.6 | physical tokens/s: 75672.4 | loss tokens/s: 54436.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.784249E-01 | lm-global-supervised-tokens: 1.213902E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.696025E-02 | lm-token-buffer-mib: 2.330172E+02 | global_load_balancing_loss: 1.001359E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11210.908 | params rms: 0.060703 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:43:37.133961 | finish at 2026-10-17 17:54:42 + [2026-10-12 17:14:46] iteration 436/ 2384 | consumed samples: 13952 | elapsed time per iteration (ms): 221371.3 | throughput per GPU (TFLOP/s/GPU): 26.5 | MFU 8.50% | FLOPs estimator: workload-aware | logical tokens/s: 75787.4 | valid tokens/s: 75787.4 | physical tokens/s: 76272.8 | loss tokens/s: 59309.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.722846E-01 | lm-global-supervised-tokens: 1.312951E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.780293E-02 | lm-token-buffer-mib: 2.329076E+02 | global_load_balancing_loss: 9.999103E-01 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11210.859 | params rms: 0.060703 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:47:11.327662 | finish at 2026-10-17 17:01:57 + [2026-10-12 17:18:30] iteration 437/ 2384 | consumed samples: 13984 | elapsed time per iteration (ms): 224512.5 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 74727.2 | valid tokens/s: 74727.2 | physical tokens/s: 75886.1 | loss tokens/s: 59940.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.857175E-01 | lm-global-supervised-tokens: 1.345736E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.880407E-02 | lm-token-buffer-mib: 2.356008E+02 | global_load_balancing_loss: 9.991078E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11210.817 | params rms: 0.060703 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:25:25.849198 | finish at 2026-10-17 18:43:56 + [2026-10-12 17:22:18] iteration 438/ 2384 | consumed samples: 14016 | elapsed time per iteration (ms): 228022.1 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.38% | FLOPs estimator: workload-aware | logical tokens/s: 73081.6 | valid tokens/s: 73081.6 | physical tokens/s: 74857.3 | loss tokens/s: 57079.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.701737E-01 | lm-global-supervised-tokens: 1.301539E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.532914E-02 | lm-token-buffer-mib: 2.375914E+02 | global_load_balancing_loss: 1.000509E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11210.782 | params rms: 0.060703 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:15:31.038095 | finish at 2026-10-17 20:37:49 + [2026-10-12 17:25:59] iteration 439/ 2384 | consumed samples: 14048 | elapsed time per iteration (ms): 220762.4 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 75996.4 | valid tokens/s: 75996.4 | physical tokens/s: 76260.2 | loss tokens/s: 57882.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.521114E-01 | lm-global-supervised-tokens: 1.277819E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.515083E-02 | lm-token-buffer-mib: 2.333677E+02 | global_load_balancing_loss: 1.002056E+00 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11210.751 | params rms: 0.060703 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:16:22.959182 | finish at 2026-10-17 16:42:22 + [2026-10-12 17:30:04] iteration 440/ 2384 | consumed samples: 14080 | elapsed time per iteration (ms): 245217.5 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.29% | FLOPs estimator: workload-aware | logical tokens/s: 68417.4 | valid tokens/s: 68417.4 | physical tokens/s: 69201.2 | loss tokens/s: 47390.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.733702E-01 | lm-global-supervised-tokens: 1.162097E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.931553E-02 | lm-token-buffer-mib: 2.335258E+02 | global_load_balancing_loss: 9.999371E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11210.722 | params rms: 0.060702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:25:02.730492 | finish at 2026-10-18 05:55:07 +times across ranks (ms): + save-checkpoint: + rank 0: 24712.71 + rank 1: 24712.73 + rank 2: 24712.74 + rank 3: 24712.71 + rank 4: 24712.73 + rank 5: 24712.72 + rank 6: 24712.73 + rank 7: 24712.71 + rank 8: 24712.57 + rank 9: 24712.67 + rank 10: 24712.83 + rank 11: 24712.56 + rank 12: 24712.80 + rank 13: 24712.74 + rank 14: 24712.56 + rank 15: 24712.57 + rank 16: 24712.39 + rank 17: 24712.40 + rank 18: 24712.40 + rank 19: 24712.40 + rank 20: 24712.36 + rank 21: 24712.39 + rank 22: 24712.39 + rank 23: 24712.39 + rank 24: 24712.78 + rank 25: 24712.79 + rank 26: 24712.78 + rank 27: 24712.77 + rank 28: 24712.75 + rank 29: 24712.82 + rank 30: 24712.80 + rank 31: 24712.81 + rank 32: 24713.06 + rank 33: 24712.97 + rank 34: 24712.88 + rank 35: 24713.01 + rank 36: 24712.93 + rank 37: 24713.00 + rank 38: 24712.90 + rank 39: 24712.73 + rank 40: 24713.04 + rank 41: 24713.02 + rank 42: 24712.94 + rank 43: 24713.03 + rank 44: 24712.98 + rank 45: 24713.00 + rank 46: 24713.00 + rank 47: 24713.03 + rank 48: 24713.22 + rank 49: 24713.13 + rank 50: 24713.29 + rank 51: 24713.24 + rank 52: 24713.07 + rank 53: 24713.18 + rank 54: 24713.09 + rank 55: 24713.07 + rank 56: 24712.71 + rank 57: 24712.73 + rank 58: 24712.74 + rank 59: 24712.73 + rank 60: 24712.74 + rank 61: 24712.71 + rank 62: 24712.71 + rank 63: 24712.69 + [2026-10-12 17:34:11] iteration 441/ 2384 | consumed samples: 14112 | elapsed time per iteration (ms): 222095.0 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 75540.6 | valid tokens/s: 75540.6 | physical tokens/s: 76244.7 | loss tokens/s: 55592.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.775919E-01 | lm-global-supervised-tokens: 1.234676E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.654020E-02 | lm-token-buffer-mib: 2.342375E+02 | global_load_balancing_loss: 9.999894E-01 | loss scale: 1.0 | grad norm: 0.312 | params norm: 11210.696 | params rms: 0.060702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:52:10.617483 | finish at 2026-10-17 17:26:22 + [2026-10-12 17:38:02] iteration 442/ 2384 | consumed samples: 14144 | elapsed time per iteration (ms): 231098.0 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 72597.8 | valid tokens/s: 72597.8 | physical tokens/s: 73733.8 | loss tokens/s: 56079.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.255384E-01 | lm-global-supervised-tokens: 1.295975E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.718400E-02 | lm-token-buffer-mib: 2.376130E+02 | global_load_balancing_loss: 1.001446E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11210.671 | params rms: 0.060702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:39:52.368416 | finish at 2026-10-17 22:17:55 + [2026-10-12 17:41:44] iteration 443/ 2384 | consumed samples: 14176 | elapsed time per iteration (ms): 221591.7 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 75712.2 | valid tokens/s: 75712.2 | physical tokens/s: 76444.5 | loss tokens/s: 56032.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.216341E-01 | lm-global-supervised-tokens: 1.241633E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.027487E-02 | lm-token-buffer-mib: 2.345766E+02 | global_load_balancing_loss: 1.000153E+00 | loss scale: 1.0 | grad norm: 0.326 | params norm: 11210.644 | params rms: 0.060702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:28:29.450051 | finish at 2026-10-17 17:10:13 + [2026-10-12 17:45:27] iteration 444/ 2384 | consumed samples: 14208 | elapsed time per iteration (ms): 222792.3 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75304.2 | valid tokens/s: 75304.2 | physical tokens/s: 76369.2 | loss tokens/s: 58411.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.577774E-01 | lm-global-supervised-tokens: 1.301360E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.349970E-02 | lm-token-buffer-mib: 2.336609E+02 | global_load_balancing_loss: 1.000210E+00 | loss scale: 1.0 | grad norm: 0.357 | params norm: 11210.619 | params rms: 0.060702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:03:37.136440 | finish at 2026-10-17 17:49:04 + [2026-10-12 17:49:08] iteration 445/ 2384 | consumed samples: 14240 | elapsed time per iteration (ms): 221664.1 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75687.5 | valid tokens/s: 75687.5 | physical tokens/s: 76133.9 | loss tokens/s: 58187.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.368866E-01 | lm-global-supervised-tokens: 1.289799E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.638139E-02 | lm-token-buffer-mib: 2.332752E+02 | global_load_balancing_loss: 1.000543E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11210.592 | params rms: 0.060702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:23:26.736922 | finish at 2026-10-17 17:12:35 + [2026-10-12 17:52:55] iteration 446/ 2384 | consumed samples: 14272 | elapsed time per iteration (ms): 226545.5 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 6.00% | FLOPs estimator: workload-aware | logical tokens/s: 74056.7 | valid tokens/s: 74056.7 | physical tokens/s: 75715.0 | loss tokens/s: 57048.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.662874E-01 | lm-global-supervised-tokens: 1.292408E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.041413E-02 | lm-token-buffer-mib: 2.395870E+02 | global_load_balancing_loss: 1.000968E+00 | loss scale: 1.0 | grad norm: 0.308 | params norm: 11210.564 | params rms: 0.060702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:57:25.119473 | finish at 2026-10-17 19:50:20 + [2026-10-12 17:56:38] iteration 447/ 2384 | consumed samples: 14304 | elapsed time per iteration (ms): 223363.4 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.02% | FLOPs estimator: workload-aware | logical tokens/s: 75111.7 | valid tokens/s: 75111.7 | physical tokens/s: 75444.4 | loss tokens/s: 56225.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.279598E-01 | lm-global-supervised-tokens: 1.255875E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.471072E-02 | lm-token-buffer-mib: 2.338156E+02 | global_load_balancing_loss: 1.000301E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11210.535 | params rms: 0.060701 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:10:54.902071 | finish at 2026-10-17 18:07:33 + [2026-10-12 18:00:21] iteration 448/ 2384 | consumed samples: 14336 | elapsed time per iteration (ms): 222921.2 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75260.6 | valid tokens/s: 75260.6 | physical tokens/s: 75960.1 | loss tokens/s: 58086.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.826558E-01 | lm-global-supervised-tokens: 1.294864E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.239443E-02 | lm-token-buffer-mib: 2.342339E+02 | global_load_balancing_loss: 1.001098E+00 | loss scale: 1.0 | grad norm: 0.374 | params norm: 11210.503 | params rms: 0.060701 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:52:55.506046 | finish at 2026-10-17 17:53:17 + [2026-10-12 18:04:04] iteration 449/ 2384 | consumed samples: 14368 | elapsed time per iteration (ms): 222415.2 | throughput per GPU (TFLOP/s/GPU): 27.1 | MFU 8.69% | FLOPs estimator: workload-aware | logical tokens/s: 75431.9 | valid tokens/s: 75431.9 | physical tokens/s: 75548.8 | loss tokens/s: 54696.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.026020E-01 | lm-global-supervised-tokens: 1.216532E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.242711E-02 | lm-token-buffer-mib: 2.324684E+02 | global_load_balancing_loss: 9.997557E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11210.467 | params rms: 0.060701 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:32:53.394459 | finish at 2026-10-17 17:36:57 + [2026-10-12 18:08:08] iteration 450/ 2384 | consumed samples: 14400 | elapsed time per iteration (ms): 244868.7 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 68515.0 | valid tokens/s: 68515.0 | physical tokens/s: 68917.6 | loss tokens/s: 53127.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.555523E-01 | lm-global-supervised-tokens: 1.300922E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.940934E-02 | lm-token-buffer-mib: 2.324163E+02 | global_load_balancing_loss: 1.001091E+00 | loss scale: 1.0 | grad norm: 0.236 | params norm: 11210.424 | params rms: 0.060701 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:32:56.157612 | finish at 2026-10-18 05:41:05 + [2026-10-12 18:11:57] iteration 451/ 2384 | consumed samples: 14432 | elapsed time per iteration (ms): 228727.5 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.58% | FLOPs estimator: workload-aware | logical tokens/s: 73350.1 | valid tokens/s: 73350.1 | physical tokens/s: 74601.8 | loss tokens/s: 50735.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.350280E-01 | lm-global-supervised-tokens: 1.160449E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.941676E-02 | lm-token-buffer-mib: 2.381002E+02 | global_load_balancing_loss: 9.994854E-01 | loss scale: 1.0 | grad norm: 0.780 | params norm: 11210.375 | params rms: 0.060701 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:48:50.162949 | finish at 2026-10-17 21:00:47 + [2026-10-12 18:15:52] iteration 452/ 2384 | consumed samples: 14464 | elapsed time per iteration (ms): 234783.9 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 71458.0 | valid tokens/s: 71458.0 | physical tokens/s: 72073.1 | loss tokens/s: 51546.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.367920E-01 | lm-global-supervised-tokens: 1.210230E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.747962E-02 | lm-token-buffer-mib: 2.347495E+02 | global_load_balancing_loss: 1.000223E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11210.326 | params rms: 0.060700 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:00:02.479182 | finish at 2026-10-18 00:15:54 + [2026-10-12 18:19:38] iteration 453/ 2384 | consumed samples: 14496 | elapsed time per iteration (ms): 226448.9 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 74088.2 | valid tokens/s: 74088.2 | physical tokens/s: 74764.1 | loss tokens/s: 57367.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.119218E-01 | lm-global-supervised-tokens: 1.299071E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.307845E+00 | lm-token-buffer-mib: 2.333520E+02 | global_load_balancing_loss: 1.000584E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11210.281 | params rms: 0.060700 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:27:52.875131 | finish at 2026-10-17 19:47:31 + [2026-10-12 18:23:24] iteration 454/ 2384 | consumed samples: 14528 | elapsed time per iteration (ms): 225915.3 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 74263.2 | valid tokens/s: 74263.2 | physical tokens/s: 75252.0 | loss tokens/s: 57748.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.360667E-01 | lm-global-supervised-tokens: 1.304630E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.408837E-02 | lm-token-buffer-mib: 2.336317E+02 | global_load_balancing_loss: 1.000963E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11210.239 | params rms: 0.060700 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:06:56.499343 | finish at 2026-10-17 19:30:21 + [2026-10-12 18:27:12] iteration 455/ 2384 | consumed samples: 14560 | elapsed time per iteration (ms): 227691.4 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 73684.0 | valid tokens/s: 73684.0 | physical tokens/s: 74502.5 | loss tokens/s: 55599.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.804605E-01 | lm-global-supervised-tokens: 1.265943E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.606216E-02 | lm-token-buffer-mib: 2.351246E+02 | global_load_balancing_loss: 1.000016E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11210.195 | params rms: 0.060700 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:00:16.779225 | finish at 2026-10-17 20:27:29 + [2026-10-12 18:30:57] iteration 456/ 2384 | consumed samples: 14592 | elapsed time per iteration (ms): 224998.0 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.74% | FLOPs estimator: workload-aware | logical tokens/s: 74566.0 | valid tokens/s: 74566.0 | physical tokens/s: 75461.8 | loss tokens/s: 59435.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.220568E-01 | lm-global-supervised-tokens: 1.337283E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.720002E-02 | lm-token-buffer-mib: 2.355744E+02 | global_load_balancing_loss: 1.001350E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11210.151 | params rms: 0.060699 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:29:56.223803 | finish at 2026-10-17 19:00:53 + [2026-10-12 18:34:39] iteration 457/ 2384 | consumed samples: 14624 | elapsed time per iteration (ms): 222386.7 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.41% | FLOPs estimator: workload-aware | logical tokens/s: 75441.3 | valid tokens/s: 75441.3 | physical tokens/s: 76175.2 | loss tokens/s: 62854.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.164437E-01 | lm-global-supervised-tokens: 1.397794E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.307570E-02 | lm-token-buffer-mib: 2.358026E+02 | global_load_balancing_loss: 1.000829E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11210.107 | params rms: 0.060699 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:02:19.144089 | finish at 2026-10-17 17:36:59 + [2026-10-12 18:38:30] iteration 458/ 2384 | consumed samples: 14656 | elapsed time per iteration (ms): 230135.2 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.39% | FLOPs estimator: workload-aware | logical tokens/s: 72901.3 | valid tokens/s: 72901.3 | physical tokens/s: 73438.6 | loss tokens/s: 54210.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.275271E-01 | lm-global-supervised-tokens: 1.247580E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.460136E-02 | lm-token-buffer-mib: 2.334305E+02 | global_load_balancing_loss: 1.001437E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11210.062 | params rms: 0.060699 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:07:20.366316 | finish at 2026-10-17 21:45:50 + [2026-10-12 18:42:09] iteration 459/ 2384 | consumed samples: 14688 | elapsed time per iteration (ms): 219354.6 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.91% | FLOPs estimator: workload-aware | logical tokens/s: 76484.3 | valid tokens/s: 76484.3 | physical tokens/s: 77130.7 | loss tokens/s: 58125.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.842088E-01 | lm-global-supervised-tokens: 1.275010E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.047560E-02 | lm-token-buffer-mib: 2.324044E+02 | global_load_balancing_loss: 1.001846E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11210.014 | params rms: 0.060699 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:17:37.509905 | finish at 2026-10-17 15:59:46 + [2026-10-12 18:46:21] iteration 460/ 2384 | consumed samples: 14720 | elapsed time per iteration (ms): 251876.5 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 66608.8 | valid tokens/s: 66608.8 | physical tokens/s: 66804.0 | loss tokens/s: 51441.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.239037E-01 | lm-global-supervised-tokens: 1.295701E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.462018E-02 | lm-token-buffer-mib: 2.327303E+02 | global_load_balancing_loss: 1.000665E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11209.965 | params rms: 0.060698 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:36:50.444965 | finish at 2026-10-18 09:23:11 + [2026-10-12 18:50:04] iteration 461/ 2384 | consumed samples: 14752 | elapsed time per iteration (ms): 223505.7 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 75063.8 | valid tokens/s: 75063.8 | physical tokens/s: 75583.1 | loss tokens/s: 57239.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.390929E-01 | lm-global-supervised-tokens: 1.279328E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.400359E-02 | lm-token-buffer-mib: 2.331534E+02 | global_load_balancing_loss: 1.000319E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11209.916 | params rms: 0.060698 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:23:21.397586 | finish at 2026-10-17 18:13:26 + [2026-10-12 18:53:49] iteration 462/ 2384 | consumed samples: 14784 | elapsed time per iteration (ms): 224628.5 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.17% | FLOPs estimator: workload-aware | logical tokens/s: 74688.7 | valid tokens/s: 74688.7 | physical tokens/s: 75854.6 | loss tokens/s: 58522.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.852581E-01 | lm-global-supervised-tokens: 1.314572E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.014391E+00 | lm-token-buffer-mib: 2.366915E+02 | global_load_balancing_loss: 1.000827E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11209.870 | params rms: 0.060698 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:55:35.937104 | finish at 2026-10-17 18:49:25 + [2026-10-12 18:57:30] iteration 463/ 2384 | consumed samples: 14816 | elapsed time per iteration (ms): 221273.6 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 75820.9 | valid tokens/s: 75820.9 | physical tokens/s: 76411.8 | loss tokens/s: 61884.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 9.269680E-01 | lm-global-supervised-tokens: 1.369339E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.286823E-02 | lm-token-buffer-mib: 2.343332E+02 | global_load_balancing_loss: 1.000073E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11209.822 | params rms: 0.060698 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:04:26.569307 | finish at 2026-10-17 17:01:57 + [2026-10-12 19:01:17] iteration 464/ 2384 | consumed samples: 14848 | elapsed time per iteration (ms): 226993.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 73910.6 | valid tokens/s: 73910.6 | physical tokens/s: 74366.1 | loss tokens/s: 55440.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.552245E-01 | lm-global-supervised-tokens: 1.258454E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.314221E-02 | lm-token-buffer-mib: 2.340548E+02 | global_load_balancing_loss: 9.988832E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11209.768 | params rms: 0.060697 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:03:46.602173 | finish at 2026-10-17 20:05:04 + [2026-10-12 19:04:59] iteration 465/ 2384 | consumed samples: 14880 | elapsed time per iteration (ms): 222248.5 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 75488.3 | valid tokens/s: 75488.3 | physical tokens/s: 76002.0 | loss tokens/s: 58492.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.614949E-01 | lm-global-supervised-tokens: 1.299979E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.589873E-02 | lm-token-buffer-mib: 2.340734E+02 | global_load_balancing_loss: 9.996758E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11209.713 | params rms: 0.060697 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:28:14.906938 | finish at 2026-10-17 17:33:14 + [2026-10-12 19:08:43] iteration 466/ 2384 | consumed samples: 14912 | elapsed time per iteration (ms): 223201.5 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 75166.2 | valid tokens/s: 75166.2 | physical tokens/s: 75743.2 | loss tokens/s: 55974.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.026620E-01 | lm-global-supervised-tokens: 1.249368E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.413862E-02 | lm-token-buffer-mib: 2.326724E+02 | global_load_balancing_loss: 1.000452E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11209.656 | params rms: 0.060697 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:55:00.435270 | finish at 2026-10-17 18:03:43 + [2026-10-12 19:12:25] iteration 467/ 2384 | consumed samples: 14944 | elapsed time per iteration (ms): 222185.2 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75510.0 | valid tokens/s: 75510.0 | physical tokens/s: 75832.4 | loss tokens/s: 53358.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.126897E-01 | lm-global-supervised-tokens: 1.185535E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.316983E-02 | lm-token-buffer-mib: 2.324176E+02 | global_load_balancing_loss: 1.000381E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11209.603 | params rms: 0.060696 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:18:49.067203 | finish at 2026-10-17 17:31:14 + [2026-10-12 19:16:15] iteration 468/ 2384 | consumed samples: 14976 | elapsed time per iteration (ms): 229740.5 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 73026.7 | valid tokens/s: 73026.7 | physical tokens/s: 73757.2 | loss tokens/s: 52906.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.113462E-01 | lm-global-supervised-tokens: 1.215485E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.714282E-02 | lm-token-buffer-mib: 2.360168E+02 | global_load_balancing_loss: 1.002094E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11209.552 | params rms: 0.060696 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:16:22.739477 | finish at 2026-10-17 21:32:37 + [2026-10-12 19:20:32] iteration 469/ 2384 | consumed samples: 15008 | elapsed time per iteration (ms): 257106.1 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.01% | FLOPs estimator: workload-aware | logical tokens/s: 65254.0 | valid tokens/s: 65254.0 | physical tokens/s: 65699.1 | loss tokens/s: 48796.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.612397E-01 | lm-global-supervised-tokens: 1.254582E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.685445E-02 | lm-token-buffer-mib: 2.336669E+02 | global_load_balancing_loss: 1.000544E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11209.503 | params rms: 0.060696 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:45:58.216814 | finish at 2026-10-18 12:06:30 + [2026-10-12 19:24:40] iteration 470/ 2384 | consumed samples: 15040 | elapsed time per iteration (ms): 248840.2 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 67421.6 | valid tokens/s: 67421.6 | physical tokens/s: 68366.6 | loss tokens/s: 51469.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.703831E-01 | lm-global-supervised-tokens: 1.280768E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.371188E-02 | lm-token-buffer-mib: 2.345078E+02 | global_load_balancing_loss: 1.000197E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11209.452 | params rms: 0.060696 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:18:00.129922 | finish at 2026-10-18 07:42:41 + [2026-10-12 19:28:21] iteration 471/ 2384 | consumed samples: 15072 | elapsed time per iteration (ms): 220754.4 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.28% | FLOPs estimator: workload-aware | logical tokens/s: 75999.3 | valid tokens/s: 75999.3 | physical tokens/s: 76328.5 | loss tokens/s: 60140.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.081863E-01 | lm-global-supervised-tokens: 1.327633E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.550766E-02 | lm-token-buffer-mib: 2.326884E+02 | global_load_balancing_loss: 9.993095E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11209.402 | params rms: 0.060695 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:18:23.152633 | finish at 2026-10-17 16:46:44 + [2026-10-12 19:32:04] iteration 472/ 2384 | consumed samples: 15104 | elapsed time per iteration (ms): 222465.1 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 75414.9 | valid tokens/s: 75414.9 | physical tokens/s: 75912.5 | loss tokens/s: 57153.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.684906E-01 | lm-global-supervised-tokens: 1.271465E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.663486E-02 | lm-token-buffer-mib: 2.343095E+02 | global_load_balancing_loss: 9.997023E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11209.350 | params rms: 0.060695 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:09:13.264002 | finish at 2026-10-17 17:41:17 + [2026-10-12 19:35:45] iteration 473/ 2384 | consumed samples: 15136 | elapsed time per iteration (ms): 221171.0 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75856.2 | valid tokens/s: 75856.2 | physical tokens/s: 76248.7 | loss tokens/s: 57000.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.388919E-01 | lm-global-supervised-tokens: 1.260683E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.788663E-02 | lm-token-buffer-mib: 2.335128E+02 | global_load_balancing_loss: 1.002853E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11209.296 | params rms: 0.060695 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:24:17.865297 | finish at 2026-10-17 17:00:03 + [2026-10-12 19:39:25] iteration 474/ 2384 | consumed samples: 15168 | elapsed time per iteration (ms): 219643.5 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 76383.8 | valid tokens/s: 76383.8 | physical tokens/s: 76690.4 | loss tokens/s: 59707.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.131408E-01 | lm-global-supervised-tokens: 1.311433E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.202061E-02 | lm-token-buffer-mib: 2.331481E+02 | global_load_balancing_loss: 1.001576E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11209.243 | params rms: 0.060694 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:31:59.122057 | finish at 2026-10-17 16:11:24 + [2026-10-12 19:43:13] iteration 475/ 2384 | consumed samples: 15200 | elapsed time per iteration (ms): 228024.1 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 73576.4 | valid tokens/s: 73576.4 | physical tokens/s: 74578.2 | loss tokens/s: 57888.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.141183E-01 | lm-global-supervised-tokens: 1.319998E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.613023E-02 | lm-token-buffer-mib: 2.372733E+02 | global_load_balancing_loss: 1.000462E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11209.189 | params rms: 0.060694 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:54:58.053449 | finish at 2026-10-17 20:38:11 + [2026-10-12 19:46:54] iteration 476/ 2384 | consumed samples: 15232 | elapsed time per iteration (ms): 221890.1 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 75610.4 | valid tokens/s: 75610.4 | physical tokens/s: 76263.5 | loss tokens/s: 57446.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.329697E-01 | lm-global-supervised-tokens: 1.274673E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.490391E-02 | lm-token-buffer-mib: 2.339839E+02 | global_load_balancing_loss: 1.001326E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11209.132 | params rms: 0.060694 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:36:06.293973 | finish at 2026-10-17 17:23:01 + [2026-10-12 19:50:39] iteration 477/ 2384 | consumed samples: 15264 | elapsed time per iteration (ms): 224673.8 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 74673.5 | valid tokens/s: 74673.5 | physical tokens/s: 75289.5 | loss tokens/s: 54128.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.186643E-01 | lm-global-supervised-tokens: 1.216116E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.487568E-02 | lm-token-buffer-mib: 2.345676E+02 | global_load_balancing_loss: 1.000987E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11209.073 | params rms: 0.060693 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:00:52.895210 | finish at 2026-10-17 18:51:32 + [2026-10-12 19:54:28] iteration 478/ 2384 | consumed samples: 15296 | elapsed time per iteration (ms): 229327.9 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 73158.2 | valid tokens/s: 73158.2 | physical tokens/s: 73541.6 | loss tokens/s: 52605.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.850888E-01 | lm-global-supervised-tokens: 1.206381E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.146975E-02 | lm-token-buffer-mib: 2.333446E+02 | global_load_balancing_loss: 9.999295E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11209.016 | params rms: 0.060693 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:24:58.990450 | finish at 2026-10-17 21:19:27 + [2026-10-12 19:58:10] iteration 479/ 2384 | consumed samples: 15328 | elapsed time per iteration (ms): 221582.4 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 75715.4 | valid tokens/s: 75715.4 | physical tokens/s: 76786.5 | loss tokens/s: 53762.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.702485E-01 | lm-global-supervised-tokens: 1.191272E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.067818E-02 | lm-token-buffer-mib: 2.344024E+02 | global_load_balancing_loss: 9.992324E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11208.963 | params rms: 0.060693 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:15:14.403577 | finish at 2026-10-17 17:13:24 + [2026-10-12 20:02:12] iteration 480/ 2384 | consumed samples: 15360 | elapsed time per iteration (ms): 242154.8 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 6.01% | FLOPs estimator: workload-aware | logical tokens/s: 69283.0 | valid tokens/s: 69283.0 | physical tokens/s: 70662.6 | loss tokens/s: 51509.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.939369E-01 | lm-global-supervised-tokens: 1.247337E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.235509E-02 | lm-token-buffer-mib: 2.338251E+02 | global_load_balancing_loss: 1.000892E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11208.908 | params rms: 0.060693 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:04:22.658733 | finish at 2026-10-18 04:06:35 +times across ranks (ms): + save-checkpoint: + rank 0: 23696.54 + rank 1: 23696.58 + rank 2: 23696.55 + rank 3: 23696.57 + rank 4: 23696.54 + rank 5: 23696.56 + rank 6: 23696.59 + rank 7: 23696.56 + rank 8: 23696.40 + rank 9: 23696.36 + rank 10: 23696.39 + rank 11: 23696.38 + rank 12: 23696.36 + rank 13: 23696.35 + rank 14: 23696.35 + rank 15: 23696.36 + rank 16: 23696.11 + rank 17: 23696.19 + rank 18: 23696.16 + rank 19: 23696.10 + rank 20: 23696.10 + rank 21: 23696.12 + rank 22: 23696.13 + rank 23: 23696.09 + rank 24: 23696.23 + rank 25: 23696.26 + rank 26: 23696.28 + rank 27: 23696.26 + rank 28: 23696.22 + rank 29: 23696.28 + rank 30: 23696.25 + rank 31: 23696.22 + rank 32: 23694.54 + rank 33: 23694.53 + rank 34: 23694.54 + rank 35: 23694.54 + rank 36: 23694.50 + rank 37: 23694.52 + rank 38: 23694.51 + rank 39: 23694.52 + rank 40: 23696.37 + rank 41: 23696.35 + rank 42: 23696.35 + rank 43: 23696.37 + rank 44: 23696.38 + rank 45: 23696.47 + rank 46: 23696.36 + rank 47: 23696.38 + rank 48: 23695.85 + rank 49: 23695.87 + rank 50: 23695.88 + rank 51: 23695.84 + rank 52: 23695.84 + rank 53: 23695.83 + rank 54: 23695.84 + rank 55: 23695.84 + rank 56: 23695.99 + rank 57: 23696.00 + rank 58: 23696.01 + rank 59: 23696.00 + rank 60: 23696.01 + rank 61: 23696.01 + rank 62: 23696.00 + rank 63: 23695.98 + [2026-10-12 20:06:18] iteration 481/ 2384 | consumed samples: 15392 | elapsed time per iteration (ms): 222166.0 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 75516.5 | valid tokens/s: 75516.5 | physical tokens/s: 76227.8 | loss tokens/s: 58839.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.822979E-01 | lm-global-supervised-tokens: 1.307217E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.351256E-01 | lm-token-buffer-mib: 2.337592E+02 | global_load_balancing_loss: 1.000712E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11208.850 | params rms: 0.060692 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:26:21.975828 | finish at 2026-10-17 17:32:40 + [2026-10-12 20:10:03] iteration 482/ 2384 | consumed samples: 15424 | elapsed time per iteration (ms): 224610.3 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 74694.7 | valid tokens/s: 74694.7 | physical tokens/s: 75425.1 | loss tokens/s: 56390.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.039103E-01 | lm-global-supervised-tokens: 1.266593E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.006542E-02 | lm-token-buffer-mib: 2.331818E+02 | global_load_balancing_loss: 1.001925E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11208.791 | params rms: 0.060692 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:40:08.781199 | finish at 2026-10-17 18:50:11 + [2026-10-12 20:13:42] iteration 483/ 2384 | consumed samples: 15456 | elapsed time per iteration (ms): 219808.0 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 76326.7 | valid tokens/s: 76326.7 | physical tokens/s: 76832.7 | loss tokens/s: 55970.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.655440E-01 | lm-global-supervised-tokens: 1.230268E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.792181E-02 | lm-token-buffer-mib: 2.341937E+02 | global_load_balancing_loss: 1.001977E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11208.733 | params rms: 0.060692 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:04:14.948793 | finish at 2026-10-17 16:17:57 + [2026-10-12 20:17:29] iteration 484/ 2384 | consumed samples: 15488 | elapsed time per iteration (ms): 226494.6 | throughput per GPU (TFLOP/s/GPU): 17.7 | MFU 5.67% | FLOPs estimator: workload-aware | logical tokens/s: 74073.3 | valid tokens/s: 74073.3 | physical tokens/s: 75543.0 | loss tokens/s: 54333.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.658140E-01 | lm-global-supervised-tokens: 1.230615E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.278933E-02 | lm-token-buffer-mib: 2.390696E+02 | global_load_balancing_loss: 1.001142E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11208.676 | params rms: 0.060691 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:32:19.682579 | finish at 2026-10-17 19:49:49 + [2026-10-12 20:21:39] iteration 485/ 2384 | consumed samples: 15520 | elapsed time per iteration (ms): 250002.9 | throughput per GPU (TFLOP/s/GPU): 17.7 | MFU 5.66% | FLOPs estimator: workload-aware | logical tokens/s: 67108.1 | valid tokens/s: 67108.1 | physical tokens/s: 67928.1 | loss tokens/s: 53333.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.014173E-01 | lm-global-supervised-tokens: 1.333344E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.385506E-02 | lm-token-buffer-mib: 2.338883E+02 | global_load_balancing_loss: 1.000777E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11208.620 | params rms: 0.060691 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:52:35.443043 | finish at 2026-10-18 08:14:14 + [2026-10-12 20:25:31] iteration 486/ 2384 | consumed samples: 15552 | elapsed time per iteration (ms): 231859.6 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 72359.2 | valid tokens/s: 72359.2 | physical tokens/s: 73020.7 | loss tokens/s: 53013.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.881733E-01 | lm-global-supervised-tokens: 1.229161E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.528977E-02 | lm-token-buffer-mib: 2.346482E+02 | global_load_balancing_loss: 9.999171E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11208.561 | params rms: 0.060691 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:14:29.541743 | finish at 2026-10-17 22:40:00 + [2026-10-12 20:29:16] iteration 487/ 2384 | consumed samples: 15584 | elapsed time per iteration (ms): 224968.0 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 74575.9 | valid tokens/s: 74575.9 | physical tokens/s: 75539.8 | loss tokens/s: 54245.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.455594E-01 | lm-global-supervised-tokens: 1.220355E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.138197E-02 | lm-token-buffer-mib: 2.354379E+02 | global_load_balancing_loss: 1.000177E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11208.500 | params rms: 0.060690 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:32:44.321205 | finish at 2026-10-17 19:02:00 + [2026-10-12 20:32:59] iteration 488/ 2384 | consumed samples: 15616 | elapsed time per iteration (ms): 222801.1 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75301.3 | valid tokens/s: 75301.3 | physical tokens/s: 76567.8 | loss tokens/s: 58118.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.520270E-01 | lm-global-supervised-tokens: 1.294881E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.742239E-02 | lm-token-buffer-mib: 2.359287E+02 | global_load_balancing_loss: 9.996201E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11208.443 | params rms: 0.060690 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:20:30.796125 | finish at 2026-10-17 17:53:29 + [2026-10-12 20:36:42] iteration 489/ 2384 | consumed samples: 15648 | elapsed time per iteration (ms): 222952.6 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.58% | FLOPs estimator: workload-aware | logical tokens/s: 75249.9 | valid tokens/s: 75249.9 | physical tokens/s: 75819.7 | loss tokens/s: 57159.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.450836E-01 | lm-global-supervised-tokens: 1.274393E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.380940E-02 | lm-token-buffer-mib: 2.347963E+02 | global_load_balancing_loss: 1.000290E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11208.389 | params rms: 0.060690 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:21:35.251100 | finish at 2026-10-17 17:58:17 + [2026-10-12 20:40:49] iteration 490/ 2384 | consumed samples: 15680 | elapsed time per iteration (ms): 247512.8 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.03% | FLOPs estimator: workload-aware | logical tokens/s: 67783.0 | valid tokens/s: 67783.0 | physical tokens/s: 68210.4 | loss tokens/s: 51791.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.254409E-01 | lm-global-supervised-tokens: 1.281906E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.698343E-02 | lm-token-buffer-mib: 2.344760E+02 | global_load_balancing_loss: 9.990963E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11208.336 | params rms: 0.060689 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:13:09.312248 | finish at 2026-10-18 06:53:58 + [2026-10-12 20:44:30] iteration 491/ 2384 | consumed samples: 15712 | elapsed time per iteration (ms): 221153.3 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.07% | FLOPs estimator: workload-aware | logical tokens/s: 75862.4 | valid tokens/s: 75862.4 | physical tokens/s: 76083.2 | loss tokens/s: 53201.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.529648E-01 | lm-global-supervised-tokens: 1.176572E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.219873E-02 | lm-token-buffer-mib: 2.330206E+02 | global_load_balancing_loss: 1.000968E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11208.285 | params rms: 0.060689 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:17:23.215042 | finish at 2026-10-17 17:01:53 + [2026-10-12 20:48:14] iteration 492/ 2384 | consumed samples: 15744 | elapsed time per iteration (ms): 223441.5 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 75085.2 | valid tokens/s: 75085.2 | physical tokens/s: 76205.2 | loss tokens/s: 57130.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.251645E-01 | lm-global-supervised-tokens: 1.276534E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.269176E-02 | lm-token-buffer-mib: 2.345852E+02 | global_load_balancing_loss: 1.000503E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11208.236 | params rms: 0.060689 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:25:51.242571 | finish at 2026-10-17 18:14:05 + [2026-10-12 20:52:07] iteration 493/ 2384 | consumed samples: 15776 | elapsed time per iteration (ms): 233625.3 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 71812.4 | valid tokens/s: 71812.4 | physical tokens/s: 72887.5 | loss tokens/s: 49405.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.329565E-01 | lm-global-supervised-tokens: 1.154239E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.872506E-02 | lm-token-buffer-mib: 2.378654E+02 | global_load_balancing_loss: 1.001008E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11208.186 | params rms: 0.060689 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:43:05.357409 | finish at 2026-10-17 23:35:13 + [2026-10-12 20:55:53] iteration 494/ 2384 | consumed samples: 15808 | elapsed time per iteration (ms): 226088.0 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.12% | FLOPs estimator: workload-aware | logical tokens/s: 74206.5 | valid tokens/s: 74206.5 | physical tokens/s: 75007.1 | loss tokens/s: 57116.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.711665E-01 | lm-global-supervised-tokens: 1.291346E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.576781E-02 | lm-token-buffer-mib: 2.346101E+02 | global_load_balancing_loss: 1.001104E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11208.140 | params rms: 0.060688 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:41:46.353457 | finish at 2026-10-17 19:37:40 + [2026-10-12 20:59:36] iteration 495/ 2384 | consumed samples: 15840 | elapsed time per iteration (ms): 222171.0 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.15% | FLOPs estimator: workload-aware | logical tokens/s: 75514.7 | valid tokens/s: 75514.7 | physical tokens/s: 76119.8 | loss tokens/s: 58809.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.805067E-01 | lm-global-supervised-tokens: 1.306579E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.002213E-02 | lm-token-buffer-mib: 2.328402E+02 | global_load_balancing_loss: 1.000055E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11208.094 | params rms: 0.060688 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:34:41.022610 | finish at 2026-10-17 17:34:17 + [2026-10-12 21:03:17] iteration 496/ 2384 | consumed samples: 15872 | elapsed time per iteration (ms): 221760.7 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 75654.6 | valid tokens/s: 75654.6 | physical tokens/s: 76126.7 | loss tokens/s: 58164.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.865644E-01 | lm-global-supervised-tokens: 1.289848E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.279540E-02 | lm-token-buffer-mib: 2.338953E+02 | global_load_balancing_loss: 1.000435E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11208.046 | params rms: 0.060688 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:18:04.255142 | finish at 2026-10-17 17:21:22 + [2026-10-12 21:06:58] iteration 497/ 2384 | consumed samples: 15904 | elapsed time per iteration (ms): 220663.4 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 76030.7 | valid tokens/s: 76030.7 | physical tokens/s: 76595.8 | loss tokens/s: 54447.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.664972E-01 | lm-global-supervised-tokens: 1.201467E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.424856E-02 | lm-token-buffer-mib: 2.328245E+02 | global_load_balancing_loss: 1.001694E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11207.997 | params rms: 0.060688 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:39:51.929255 | finish at 2026-10-17 16:46:50 + [2026-10-12 21:10:43] iteration 498/ 2384 | consumed samples: 15936 | elapsed time per iteration (ms): 224801.7 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 74631.0 | valid tokens/s: 74631.0 | physical tokens/s: 75665.7 | loss tokens/s: 56867.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.336050E-01 | lm-global-supervised-tokens: 1.278383E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.375364E-02 | lm-token-buffer-mib: 2.370899E+02 | global_load_balancing_loss: 1.000233E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11207.949 | params rms: 0.060687 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:46:16.089604 | finish at 2026-10-17 18:56:59 + [2026-10-12 21:14:30] iteration 499/ 2384 | consumed samples: 15968 | elapsed time per iteration (ms): 227401.3 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.05% | FLOPs estimator: workload-aware | logical tokens/s: 73777.8 | valid tokens/s: 73777.8 | physical tokens/s: 74254.5 | loss tokens/s: 55078.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.206359E-01 | lm-global-supervised-tokens: 1.252496E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.253661E-02 | lm-token-buffer-mib: 2.337034E+02 | global_load_balancing_loss: 9.999521E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11207.905 | params rms: 0.060687 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:04:11.455805 | finish at 2026-10-17 20:18:42 + [2026-10-12 21:18:36] iteration 500/ 2384 | consumed samples: 16000 | elapsed time per iteration (ms): 245566.0 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 68320.6 | valid tokens/s: 68320.6 | physical tokens/s: 68991.2 | loss tokens/s: 49675.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.416401E-01 | lm-global-supervised-tokens: 1.219856E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.985525E-02 | lm-token-buffer-mib: 2.354858E+02 | global_load_balancing_loss: 9.999008E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11207.860 | params rms: 0.060687 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:30:46.413594 | finish at 2026-10-18 05:49:22 + [2026-10-12 21:22:16] iteration 501/ 2384 | consumed samples: 16032 | elapsed time per iteration (ms): 220608.2 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.19% | FLOPs estimator: workload-aware | logical tokens/s: 76049.7 | valid tokens/s: 76049.7 | physical tokens/s: 76574.6 | loss tokens/s: 60529.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.600543E-01 | lm-global-supervised-tokens: 1.335320E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.726959E-02 | lm-token-buffer-mib: 2.341679E+02 | global_load_balancing_loss: 1.000535E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11207.813 | params rms: 0.060687 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:23:25.222780 | finish at 2026-10-17 16:45:42 + [2026-10-12 21:25:57] iteration 502/ 2384 | consumed samples: 16064 | elapsed time per iteration (ms): 221005.9 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 75912.9 | valid tokens/s: 75912.8 | physical tokens/s: 76749.6 | loss tokens/s: 60172.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.734245E-01 | lm-global-supervised-tokens: 1.329838E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.417998E-02 | lm-token-buffer-mib: 2.350063E+02 | global_load_balancing_loss: 1.000218E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11207.765 | params rms: 0.060686 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:32:13.177660 | finish at 2026-10-17 16:58:11 + [2026-10-12 21:29:39] iteration 503/ 2384 | consumed samples: 16096 | elapsed time per iteration (ms): 222013.5 | throughput per GPU (TFLOP/s/GPU): 26.5 | MFU 8.49% | FLOPs estimator: workload-aware | logical tokens/s: 75568.3 | valid tokens/s: 75568.3 | physical tokens/s: 75841.4 | loss tokens/s: 60173.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.745273E-01 | lm-global-supervised-tokens: 1.335941E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.144154E-02 | lm-token-buffer-mib: 2.325118E+02 | global_load_balancing_loss: 1.000416E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11207.719 | params rms: 0.060686 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:00:07.354437 | finish at 2026-10-17 17:29:47 + [2026-10-12 21:33:26] iteration 504/ 2384 | consumed samples: 16128 | elapsed time per iteration (ms): 226309.4 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 74133.8 | valid tokens/s: 74133.8 | physical tokens/s: 74748.5 | loss tokens/s: 53595.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.664813E-01 | lm-global-supervised-tokens: 1.212912E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.372142E-02 | lm-token-buffer-mib: 2.350351E+02 | global_load_balancing_loss: 1.000516E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11207.675 | params rms: 0.060686 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:11:01.625538 | finish at 2026-10-17 19:44:27 + [2026-10-12 21:37:09] iteration 505/ 2384 | consumed samples: 16160 | elapsed time per iteration (ms): 223218.6 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75160.4 | valid tokens/s: 75160.4 | physical tokens/s: 75785.2 | loss tokens/s: 62627.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.368588E-01 | lm-global-supervised-tokens: 1.397971E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.651759E-02 | lm-token-buffer-mib: 2.339457E+02 | global_load_balancing_loss: 1.002142E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11207.629 | params rms: 0.060686 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:30:27.816216 | finish at 2026-10-17 18:07:37 + [2026-10-12 21:40:54] iteration 506/ 2384 | consumed samples: 16192 | elapsed time per iteration (ms): 224832.3 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 74620.9 | valid tokens/s: 74620.9 | physical tokens/s: 75284.0 | loss tokens/s: 53833.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.057990E-01 | lm-global-supervised-tokens: 1.210355E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.450466E-02 | lm-token-buffer-mib: 2.343804E+02 | global_load_balancing_loss: 9.998326E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11207.583 | params rms: 0.060685 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:17:14.972438 | finish at 2026-10-17 18:58:09 + [2026-10-12 21:44:50] iteration 507/ 2384 | consumed samples: 16224 | elapsed time per iteration (ms): 236316.3 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 70994.7 | valid tokens/s: 70994.7 | physical tokens/s: 71539.9 | loss tokens/s: 56110.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.176676E-01 | lm-global-supervised-tokens: 1.325974E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.808094E-02 | lm-token-buffer-mib: 2.337849E+02 | global_load_balancing_loss: 9.992426E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11207.539 | params rms: 0.060685 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:12:45.626919 | finish at 2026-10-18 00:57:36 + [2026-10-12 21:48:38] iteration 508/ 2384 | consumed samples: 16256 | elapsed time per iteration (ms): 227496.3 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 73747.1 | valid tokens/s: 73747.1 | physical tokens/s: 74477.0 | loss tokens/s: 52161.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.131212E-01 | lm-global-supervised-tokens: 1.186645E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.170354E+00 | lm-token-buffer-mib: 2.347731E+02 | global_load_balancing_loss: 9.990718E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11207.494 | params rms: 0.060685 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:33:02.985414 | finish at 2026-10-17 20:21:41 + [2026-10-12 21:52:20] iteration 509/ 2384 | consumed samples: 16288 | elapsed time per iteration (ms): 222012.5 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 75568.6 | valid tokens/s: 75568.6 | physical tokens/s: 76348.8 | loss tokens/s: 60094.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.128570E-01 | lm-global-supervised-tokens: 1.334174E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.524671E-02 | lm-token-buffer-mib: 2.336554E+02 | global_load_balancing_loss: 1.000981E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11207.446 | params rms: 0.060685 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:37:53.527443 | finish at 2026-10-17 17:30:13 + [2026-10-12 21:56:27] iteration 510/ 2384 | consumed samples: 16320 | elapsed time per iteration (ms): 247006.6 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.97% | FLOPs estimator: workload-aware | logical tokens/s: 67922.0 | valid tokens/s: 67922.0 | physical tokens/s: 69061.4 | loss tokens/s: 51239.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.986166E-01 | lm-global-supervised-tokens: 1.265636E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.536961E-02 | lm-token-buffer-mib: 2.387008E+02 | global_load_balancing_loss: 1.000483E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11207.402 | params rms: 0.060684 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:34:50.412004 | finish at 2026-10-18 06:31:17 + [2026-10-12 22:00:08] iteration 511/ 2384 | consumed samples: 16352 | elapsed time per iteration (ms): 221104.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75879.1 | valid tokens/s: 75879.1 | physical tokens/s: 76515.9 | loss tokens/s: 54169.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.122568E-01 | lm-global-supervised-tokens: 1.197706E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.800898E-02 | lm-token-buffer-mib: 2.333760E+02 | global_load_balancing_loss: 1.000819E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11207.356 | params rms: 0.060684 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:02:08.665639 | finish at 2026-10-17 17:02:16 + [2026-10-12 22:03:48] iteration 512/ 2384 | consumed samples: 16384 | elapsed time per iteration (ms): 220562.5 | throughput per GPU (TFLOP/s/GPU): 26.1 | MFU 8.37% | FLOPs estimator: workload-aware | logical tokens/s: 76065.5 | valid tokens/s: 76065.5 | physical tokens/s: 76659.9 | loss tokens/s: 53539.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.384848E-01 | lm-global-supervised-tokens: 1.180873E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.790494E-02 | lm-token-buffer-mib: 2.340044E+02 | global_load_balancing_loss: 1.001927E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11207.308 | params rms: 0.060684 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:41:32.972328 | finish at 2026-10-17 16:45:21 + [2026-10-12 22:07:32] iteration 513/ 2384 | consumed samples: 16416 | elapsed time per iteration (ms): 223992.7 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 74900.5 | valid tokens/s: 74900.5 | physical tokens/s: 75578.2 | loss tokens/s: 57021.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.096708E-01 | lm-global-supervised-tokens: 1.277248E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.297661E-02 | lm-token-buffer-mib: 2.347298E+02 | global_load_balancing_loss: 1.001364E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11207.254 | params rms: 0.060684 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:24:50.420397 | finish at 2026-10-17 18:32:23 + [2026-10-12 22:11:16] iteration 514/ 2384 | consumed samples: 16448 | elapsed time per iteration (ms): 223911.8 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.12% | FLOPs estimator: workload-aware | logical tokens/s: 74927.6 | valid tokens/s: 74927.6 | physical tokens/s: 75881.6 | loss tokens/s: 58664.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.906469E-01 | lm-global-supervised-tokens: 1.313558E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.780306E-02 | lm-token-buffer-mib: 2.360000E+02 | global_load_balancing_loss: 1.002096E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11207.202 | params rms: 0.060683 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:18:35.010543 | finish at 2026-10-17 18:29:51 + [2026-10-12 22:14:57] iteration 515/ 2384 | consumed samples: 16480 | elapsed time per iteration (ms): 220438.9 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 76108.2 | valid tokens/s: 76108.2 | physical tokens/s: 77040.2 | loss tokens/s: 60440.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.998107E-01 | lm-global-supervised-tokens: 1.332347E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.992461E-02 | lm-token-buffer-mib: 2.344453E+02 | global_load_balancing_loss: 1.000127E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11207.157 | params rms: 0.060683 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:26:40.364691 | finish at 2026-10-17 16:41:37 + [2026-10-12 22:18:39] iteration 516/ 2384 | consumed samples: 16512 | elapsed time per iteration (ms): 221978.9 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 75580.2 | valid tokens/s: 75580.1 | physical tokens/s: 76411.5 | loss tokens/s: 58445.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.434105E-01 | lm-global-supervised-tokens: 1.297364E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.354340E-02 | lm-token-buffer-mib: 2.347692E+02 | global_load_balancing_loss: 9.997735E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11207.113 | params rms: 0.060683 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:10:56.499607 | finish at 2026-10-17 17:29:35 + [2026-10-12 22:22:18] iteration 517/ 2384 | consumed samples: 16544 | elapsed time per iteration (ms): 219158.6 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.86% | FLOPs estimator: workload-aware | logical tokens/s: 76552.6 | valid tokens/s: 76552.6 | physical tokens/s: 76981.6 | loss tokens/s: 57514.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.276724E-01 | lm-global-supervised-tokens: 1.260482E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.711278E-02 | lm-token-buffer-mib: 2.333017E+02 | global_load_balancing_loss: 1.000012E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11207.066 | params rms: 0.060683 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:39:29.068091 | finish at 2026-10-17 16:01:47 + [2026-10-12 22:26:02] iteration 518/ 2384 | consumed samples: 16576 | elapsed time per iteration (ms): 224104.3 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 74863.4 | valid tokens/s: 74863.4 | physical tokens/s: 75948.9 | loss tokens/s: 53626.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.423633E-01 | lm-global-supervised-tokens: 1.201796E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.392706E-02 | lm-token-buffer-mib: 2.360547E+02 | global_load_balancing_loss: 9.998465E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11207.018 | params rms: 0.060682 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:09:38.655873 | finish at 2026-10-17 18:35:40 + [2026-10-12 22:29:45] iteration 519/ 2384 | consumed samples: 16608 | elapsed time per iteration (ms): 223619.0 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75025.7 | valid tokens/s: 75025.7 | physical tokens/s: 75749.4 | loss tokens/s: 56649.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.116921E-01 | lm-global-supervised-tokens: 1.266784E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.346421E-02 | lm-token-buffer-mib: 2.331627E+02 | global_load_balancing_loss: 1.000732E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11206.968 | params rms: 0.060682 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:50:49.355774 | finish at 2026-10-17 18:20:35 + [2026-10-12 22:33:49] iteration 520/ 2384 | consumed samples: 16640 | elapsed time per iteration (ms): 243547.4 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 68886.8 | valid tokens/s: 68886.8 | physical tokens/s: 69298.1 | loss tokens/s: 51017.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.091291E-01 | lm-global-supervised-tokens: 1.242518E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.626329E-02 | lm-token-buffer-mib: 2.343984E+02 | global_load_balancing_loss: 1.000955E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11206.916 | params rms: 0.060682 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:06:12.426924 | finish at 2026-10-18 04:40:01 +times across ranks (ms): + save-checkpoint: + rank 0: 25341.94 + rank 1: 25341.92 + rank 2: 25341.93 + rank 3: 25341.91 + rank 4: 25341.90 + rank 5: 25341.92 + rank 6: 25341.96 + rank 7: 25341.91 + rank 8: 25341.25 + rank 9: 25341.46 + rank 10: 25341.49 + rank 11: 25341.39 + rank 12: 25341.41 + rank 13: 25341.39 + rank 14: 25341.46 + rank 15: 25341.41 + rank 16: 25341.91 + rank 17: 25341.90 + rank 18: 25341.91 + rank 19: 25341.91 + rank 20: 25341.87 + rank 21: 25341.91 + rank 22: 25341.96 + rank 23: 25341.87 + rank 24: 25341.80 + rank 25: 25341.93 + rank 26: 25341.88 + rank 27: 25341.80 + rank 28: 25341.90 + rank 29: 25341.94 + rank 30: 25341.89 + rank 31: 25341.81 + rank 32: 25341.46 + rank 33: 25341.52 + rank 34: 25341.45 + rank 35: 25341.47 + rank 36: 25341.48 + rank 37: 25341.45 + rank 38: 25341.47 + rank 39: 25341.48 + rank 40: 25341.59 + rank 41: 25341.59 + rank 42: 25341.58 + rank 43: 25341.60 + rank 44: 25341.62 + rank 45: 25341.56 + rank 46: 25341.55 + rank 47: 25341.57 + rank 48: 25341.82 + rank 49: 25341.87 + rank 50: 25341.84 + rank 51: 25341.78 + rank 52: 25341.77 + rank 53: 25341.78 + rank 54: 25341.78 + rank 55: 25341.80 + rank 56: 25341.81 + rank 57: 25341.88 + rank 58: 25341.82 + rank 59: 25341.84 + rank 60: 25341.85 + rank 61: 25341.84 + rank 62: 25341.82 + rank 63: 25341.81 + [2026-10-12 22:37:56] iteration 521/ 2384 | consumed samples: 16672 | elapsed time per iteration (ms): 221926.2 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.24% | FLOPs estimator: workload-aware | logical tokens/s: 75598.1 | valid tokens/s: 75598.1 | physical tokens/s: 76587.9 | loss tokens/s: 58895.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.498210E-01 | lm-global-supervised-tokens: 1.307045E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.336671E-01 | lm-token-buffer-mib: 2.354704E+02 | global_load_balancing_loss: 1.000759E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11206.863 | params rms: 0.060682 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:50:48.447809 | finish at 2026-10-17 17:28:45 + [2026-10-12 22:41:41] iteration 522/ 2384 | consumed samples: 16704 | elapsed time per iteration (ms): 224898.3 | throughput per GPU (TFLOP/s/GPU): 18.1 | MFU 5.80% | FLOPs estimator: workload-aware | logical tokens/s: 74599.0 | valid tokens/s: 74599.0 | physical tokens/s: 75187.3 | loss tokens/s: 56535.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.649239E-01 | lm-global-supervised-tokens: 1.271476E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.830069E-02 | lm-token-buffer-mib: 2.332015E+02 | global_load_balancing_loss: 1.000018E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11206.814 | params rms: 0.060681 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:19:20.712607 | finish at 2026-10-17 19:01:02 + [2026-10-12 22:45:20] iteration 523/ 2384 | consumed samples: 16736 | elapsed time per iteration (ms): 219106.1 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 76571.1 | valid tokens/s: 76571.1 | physical tokens/s: 77575.1 | loss tokens/s: 58756.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.348522E-01 | lm-global-supervised-tokens: 1.287387E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.969560E-02 | lm-token-buffer-mib: 2.342439E+02 | global_load_balancing_loss: 9.996264E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11206.767 | params rms: 0.060681 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:15:56.488636 | finish at 2026-10-17 16:01:17 + [2026-10-12 22:49:02] iteration 524/ 2384 | consumed samples: 16768 | elapsed time per iteration (ms): 221576.5 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75717.3 | valid tokens/s: 75717.3 | physical tokens/s: 77119.5 | loss tokens/s: 58467.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.114246E-01 | lm-global-supervised-tokens: 1.295499E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.368531E-02 | lm-token-buffer-mib: 2.342901E+02 | global_load_balancing_loss: 9.998070E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11206.719 | params rms: 0.060681 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:28:52.226028 | finish at 2026-10-17 17:17:54 + [2026-10-12 22:52:43] iteration 525/ 2384 | consumed samples: 16800 | elapsed time per iteration (ms): 221346.8 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.83% | FLOPs estimator: workload-aware | logical tokens/s: 75795.9 | valid tokens/s: 75795.9 | physical tokens/s: 76501.0 | loss tokens/s: 58876.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.492880E-01 | lm-global-supervised-tokens: 1.303215E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.201778E-02 | lm-token-buffer-mib: 2.339994E+02 | global_load_balancing_loss: 1.001479E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11206.671 | params rms: 0.060680 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:18:03.775826 | finish at 2026-10-17 17:10:47 + [2026-10-12 22:56:33] iteration 526/ 2384 | consumed samples: 16832 | elapsed time per iteration (ms): 230016.3 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 72939.1 | valid tokens/s: 72939.1 | physical tokens/s: 74083.1 | loss tokens/s: 55452.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.857777E-01 | lm-global-supervised-tokens: 1.275493E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.048927E-02 | lm-token-buffer-mib: 2.370938E+02 | global_load_balancing_loss: 9.993433E-01 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11206.626 | params rms: 0.060680 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:42:50.296406 | finish at 2026-10-17 21:39:24 + [2026-10-12 23:00:19] iteration 527/ 2384 | consumed samples: 16864 | elapsed time per iteration (ms): 225835.3 | throughput per GPU (TFLOP/s/GPU): 26.1 | MFU 8.35% | FLOPs estimator: workload-aware | logical tokens/s: 74289.4 | valid tokens/s: 74289.4 | physical tokens/s: 75049.1 | loss tokens/s: 55936.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.560523E-01 | lm-global-supervised-tokens: 1.263250E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.400226E-02 | lm-token-buffer-mib: 2.335589E+02 | global_load_balancing_loss: 1.000697E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11206.580 | params rms: 0.060680 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:29:36.159126 | finish at 2026-10-17 19:29:55 + [2026-10-12 23:04:02] iteration 528/ 2384 | consumed samples: 16896 | elapsed time per iteration (ms): 223031.4 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 75223.5 | valid tokens/s: 75223.5 | physical tokens/s: 75852.3 | loss tokens/s: 58006.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.989670E-01 | lm-global-supervised-tokens: 1.293729E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.412241E-02 | lm-token-buffer-mib: 2.327823E+02 | global_load_balancing_loss: 1.001035E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11206.530 | params rms: 0.060680 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:59:06.305328 | finish at 2026-10-17 18:03:08 + [2026-10-12 23:07:45] iteration 529/ 2384 | consumed samples: 16928 | elapsed time per iteration (ms): 223107.7 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.02% | FLOPs estimator: workload-aware | logical tokens/s: 75197.8 | valid tokens/s: 75197.8 | physical tokens/s: 76157.9 | loss tokens/s: 56170.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.913446E-01 | lm-global-supervised-tokens: 1.253210E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.474439E-02 | lm-token-buffer-mib: 2.350242E+02 | global_load_balancing_loss: 1.000469E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11206.476 | params rms: 0.060679 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:57:44.869061 | finish at 2026-10-17 18:05:30 + [2026-10-12 23:11:49] iteration 530/ 2384 | consumed samples: 16960 | elapsed time per iteration (ms): 243366.5 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.24% | FLOPs estimator: workload-aware | logical tokens/s: 68937.9 | valid tokens/s: 68937.9 | physical tokens/s: 69556.3 | loss tokens/s: 53975.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.686459E-01 | lm-global-supervised-tokens: 1.313579E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.316258E-02 | lm-token-buffer-mib: 2.336134E+02 | global_load_balancing_loss: 1.001421E+00 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11206.424 | params rms: 0.060679 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:20:01.560657 | finish at 2026-10-18 04:31:50 + [2026-10-12 23:15:32] iteration 531/ 2384 | consumed samples: 16992 | elapsed time per iteration (ms): 223475.3 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 75074.0 | valid tokens/s: 75074.0 | physical tokens/s: 76204.5 | loss tokens/s: 57337.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.276588E-01 | lm-global-supervised-tokens: 1.281359E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.908812E-02 | lm-token-buffer-mib: 2.352258E+02 | global_load_balancing_loss: 1.000230E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11206.372 | params rms: 0.060679 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:01:39.817627 | finish at 2026-10-17 18:17:12 + [2026-10-12 23:19:15] iteration 532/ 2384 | consumed samples: 17024 | elapsed time per iteration (ms): 223307.0 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 75130.5 | valid tokens/s: 75130.5 | physical tokens/s: 75991.1 | loss tokens/s: 51776.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.340519E-01 | lm-global-supervised-tokens: 1.156214E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.313534E-02 | lm-token-buffer-mib: 2.332983E+02 | global_load_balancing_loss: 1.001107E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11206.319 | params rms: 0.060679 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:52:44.494090 | finish at 2026-10-17 18:12:00 + [2026-10-12 23:23:00] iteration 533/ 2384 | consumed samples: 17056 | elapsed time per iteration (ms): 224920.9 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 74591.5 | valid tokens/s: 74591.5 | physical tokens/s: 75479.4 | loss tokens/s: 60639.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.372313E-01 | lm-global-supervised-tokens: 1.363918E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.349945E-02 | lm-token-buffer-mib: 2.343631E+02 | global_load_balancing_loss: 1.000593E+00 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11206.268 | params rms: 0.060678 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:38:48.562707 | finish at 2026-10-17 19:01:49 + [2026-10-12 23:26:45] iteration 534/ 2384 | consumed samples: 17088 | elapsed time per iteration (ms): 224743.8 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 74650.3 | valid tokens/s: 74650.3 | physical tokens/s: 75477.4 | loss tokens/s: 56131.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.324072E-01 | lm-global-supervised-tokens: 1.261528E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.291697E-02 | lm-token-buffer-mib: 2.353121E+02 | global_load_balancing_loss: 1.000595E+00 | loss scale: 1.0 | grad norm: 0.234 | params norm: 11206.217 | params rms: 0.060678 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:29:36.037800 | finish at 2026-10-17 18:56:21 + [2026-10-12 23:30:27] iteration 535/ 2384 | consumed samples: 17120 | elapsed time per iteration (ms): 221993.3 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75575.1 | valid tokens/s: 75575.1 | physical tokens/s: 76569.9 | loss tokens/s: 58185.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.913007E-01 | lm-global-supervised-tokens: 1.291683E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.280537E-02 | lm-token-buffer-mib: 2.343457E+02 | global_load_balancing_loss: 1.000872E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11206.166 | params rms: 0.060678 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:01:05.690538 | finish at 2026-10-17 17:31:33 + [2026-10-12 23:34:08] iteration 536/ 2384 | consumed samples: 17152 | elapsed time per iteration (ms): 220913.0 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75944.9 | valid tokens/s: 75944.9 | physical tokens/s: 76213.7 | loss tokens/s: 52908.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.056991E-01 | lm-global-supervised-tokens: 1.168826E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.289793E-02 | lm-token-buffer-mib: 2.332700E+02 | global_load_balancing_loss: 9.999356E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11206.116 | params rms: 0.060677 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:24:07.227722 | finish at 2026-10-17 16:58:15 + [2026-10-12 23:37:48] iteration 537/ 2384 | consumed samples: 17184 | elapsed time per iteration (ms): 219740.6 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 76349.9 | valid tokens/s: 76349.9 | physical tokens/s: 76900.3 | loss tokens/s: 57513.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.233408E-01 | lm-global-supervised-tokens: 1.263803E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.516937E-02 | lm-token-buffer-mib: 2.339165E+02 | global_load_balancing_loss: 1.000194E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11206.065 | params rms: 0.060677 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:44:20.819265 | finish at 2026-10-17 16:22:08 + [2026-10-12 23:41:34] iteration 538/ 2384 | consumed samples: 17216 | elapsed time per iteration (ms): 225951.5 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.25% | FLOPs estimator: workload-aware | logical tokens/s: 74251.3 | valid tokens/s: 74251.3 | physical tokens/s: 75233.4 | loss tokens/s: 62957.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.365298E-01 | lm-global-supervised-tokens: 1.422529E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.641535E-02 | lm-token-buffer-mib: 2.325475E+02 | global_load_balancing_loss: 9.998648E-01 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11206.014 | params rms: 0.060677 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:51:46.478130 | finish at 2026-10-17 19:33:20 + [2026-10-12 23:45:16] iteration 539/ 2384 | consumed samples: 17248 | elapsed time per iteration (ms): 222292.0 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.48% | FLOPs estimator: workload-aware | logical tokens/s: 75473.6 | valid tokens/s: 75473.6 | physical tokens/s: 76903.8 | loss tokens/s: 57167.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.238246E-01 | lm-global-supervised-tokens: 1.270792E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.772165E-02 | lm-token-buffer-mib: 2.365138E+02 | global_load_balancing_loss: 1.000431E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11205.965 | params rms: 0.060677 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:55:28.741862 | finish at 2026-10-17 17:40:45 + [2026-10-12 23:49:23] iteration 540/ 2384 | consumed samples: 17280 | elapsed time per iteration (ms): 247019.8 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 67918.4 | valid tokens/s: 67918.4 | physical tokens/s: 68234.3 | loss tokens/s: 53331.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.037701E-01 | lm-global-supervised-tokens: 1.317397E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.460850E-02 | lm-token-buffer-mib: 2.330017E+02 | global_load_balancing_loss: 9.991301E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11205.916 | params rms: 0.060676 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:31:44.479010 | finish at 2026-10-18 06:21:07 + [2026-10-12 23:53:10] iteration 541/ 2384 | consumed samples: 17312 | elapsed time per iteration (ms): 226925.6 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.70% | FLOPs estimator: workload-aware | logical tokens/s: 73932.6 | valid tokens/s: 73932.6 | physical tokens/s: 74399.7 | loss tokens/s: 58169.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.877312E-01 | lm-global-supervised-tokens: 1.320010E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.683607E-02 | lm-token-buffer-mib: 2.337157E+02 | global_load_balancing_loss: 9.998785E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11205.871 | params rms: 0.060676 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:10:23.851455 | finish at 2026-10-17 20:03:34 + [2026-10-12 23:56:50] iteration 542/ 2384 | consumed samples: 17344 | elapsed time per iteration (ms): 220462.5 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.98% | FLOPs estimator: workload-aware | logical tokens/s: 76100.0 | valid tokens/s: 76100.0 | physical tokens/s: 76693.7 | loss tokens/s: 57440.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.406320E-01 | lm-global-supervised-tokens: 1.266354E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.881115E-02 | lm-token-buffer-mib: 2.336283E+02 | global_load_balancing_loss: 1.000719E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11205.821 | params rms: 0.060676 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:48:11.946695 | finish at 2026-10-17 16:45:02 + [2026-10-13 00:00:32] iteration 543/ 2384 | consumed samples: 17376 | elapsed time per iteration (ms): 221877.7 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.93% | FLOPs estimator: workload-aware | logical tokens/s: 75614.4 | valid tokens/s: 75614.4 | physical tokens/s: 76448.3 | loss tokens/s: 59010.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.294773E-01 | lm-global-supervised-tokens: 1.309317E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.271645E-02 | lm-token-buffer-mib: 2.354771E+02 | global_load_balancing_loss: 1.002396E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11205.770 | params rms: 0.060676 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:27:56.901610 | finish at 2026-10-17 17:28:29 + [2026-10-13 00:04:14] iteration 544/ 2384 | consumed samples: 17408 | elapsed time per iteration (ms): 221787.2 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 75645.5 | valid tokens/s: 75645.5 | physical tokens/s: 76555.4 | loss tokens/s: 59162.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.189779E-01 | lm-global-supervised-tokens: 1.312146E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.469428E-02 | lm-token-buffer-mib: 2.339407E+02 | global_load_balancing_loss: 1.001728E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11205.719 | params rms: 0.060675 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:21:28.502789 | finish at 2026-10-17 17:25:42 + [2026-10-13 00:07:55] iteration 545/ 2384 | consumed samples: 17440 | elapsed time per iteration (ms): 221186.5 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.58% | FLOPs estimator: workload-aware | logical tokens/s: 75850.9 | valid tokens/s: 75850.9 | physical tokens/s: 76329.6 | loss tokens/s: 59121.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.992651E-01 | lm-global-supervised-tokens: 1.307696E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.794073E-02 | lm-token-buffer-mib: 2.338007E+02 | global_load_balancing_loss: 1.001729E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11205.673 | params rms: 0.060675 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:59:22.024932 | finish at 2026-10-17 17:07:17 + [2026-10-13 00:11:41] iteration 546/ 2384 | consumed samples: 17472 | elapsed time per iteration (ms): 226254.2 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 74152.0 | valid tokens/s: 74152.0 | physical tokens/s: 75231.8 | loss tokens/s: 55441.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.032297E-01 | lm-global-supervised-tokens: 1.254393E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.751399E-02 | lm-token-buffer-mib: 2.347231E+02 | global_load_balancing_loss: 9.996946E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11205.632 | params rms: 0.060675 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:30:55.202038 | finish at 2026-10-17 19:42:37 + [2026-10-13 00:15:27] iteration 547/ 2384 | consumed samples: 17504 | elapsed time per iteration (ms): 225507.7 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 74397.3 | valid tokens/s: 74397.3 | physical tokens/s: 75148.5 | loss tokens/s: 52952.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.529186E-01 | lm-global-supervised-tokens: 1.194122E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.315633E-02 | lm-token-buffer-mib: 2.348485E+02 | global_load_balancing_loss: 9.992157E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11205.590 | params rms: 0.060675 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:04:17.659729 | finish at 2026-10-17 19:19:45 + [2026-10-13 00:19:08] iteration 548/ 2384 | consumed samples: 17536 | elapsed time per iteration (ms): 221258.2 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 75826.3 | valid tokens/s: 75826.3 | physical tokens/s: 76600.2 | loss tokens/s: 61736.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.932269E-01 | lm-global-supervised-tokens: 1.365962E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.705901E-01 | lm-token-buffer-mib: 2.341475E+02 | global_load_balancing_loss: 9.997584E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11205.546 | params rms: 0.060674 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:50:30.099721 | finish at 2026-10-17 17:09:38 + [2026-10-13 00:22:51] iteration 549/ 2384 | consumed samples: 17568 | elapsed time per iteration (ms): 222642.6 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 75354.7 | valid tokens/s: 75354.7 | physical tokens/s: 75745.3 | loss tokens/s: 55716.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.482076E-01 | lm-global-supervised-tokens: 1.240486E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.480363E-02 | lm-token-buffer-mib: 2.332019E+02 | global_load_balancing_loss: 9.994490E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11205.507 | params rms: 0.060674 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:29:09.084047 | finish at 2026-10-17 17:52:00 + [2026-10-13 00:26:56] iteration 550/ 2384 | consumed samples: 17600 | elapsed time per iteration (ms): 244689.8 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 68565.2 | valid tokens/s: 68565.2 | physical tokens/s: 69478.9 | loss tokens/s: 50400.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.747700E-01 | lm-global-supervised-tokens: 1.233237E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.329032E-02 | lm-token-buffer-mib: 2.341783E+02 | global_load_balancing_loss: 1.000277E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11205.466 | params rms: 0.060674 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:39:21.016724 | finish at 2026-10-18 05:06:17 + [2026-10-13 00:30:38] iteration 551/ 2384 | consumed samples: 17632 | elapsed time per iteration (ms): 222566.8 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 75380.5 | valid tokens/s: 75380.5 | physical tokens/s: 76284.9 | loss tokens/s: 58658.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.062754E-01 | lm-global-supervised-tokens: 1.305550E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.622220E-02 | lm-token-buffer-mib: 2.332708E+02 | global_load_balancing_loss: 1.000397E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11205.427 | params rms: 0.060674 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:19:24.973896 | finish at 2026-10-17 17:50:03 + [2026-10-13 00:34:18] iteration 552/ 2384 | consumed samples: 17664 | elapsed time per iteration (ms): 220211.9 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 76186.5 | valid tokens/s: 76186.5 | physical tokens/s: 76497.7 | loss tokens/s: 60544.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.382173E-01 | lm-global-supervised-tokens: 1.333271E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.819158E-02 | lm-token-buffer-mib: 2.329470E+02 | global_load_balancing_loss: 1.000807E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11205.386 | params rms: 0.060674 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:03:48.188562 | finish at 2026-10-17 16:38:07 + [2026-10-13 00:38:00] iteration 553/ 2384 | consumed samples: 17696 | elapsed time per iteration (ms): 221222.8 | throughput per GPU (TFLOP/s/GPU): 26.5 | MFU 8.51% | FLOPs estimator: workload-aware | logical tokens/s: 75838.5 | valid tokens/s: 75838.5 | physical tokens/s: 76047.1 | loss tokens/s: 56027.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.101289E-01 | lm-global-supervised-tokens: 1.239466E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.267823E-02 | lm-token-buffer-mib: 2.322669E+02 | global_load_balancing_loss: 1.000680E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11205.341 | params rms: 0.060673 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:30:58.874364 | finish at 2026-10-17 17:08:58 + [2026-10-13 00:41:46] iteration 554/ 2384 | consumed samples: 17728 | elapsed time per iteration (ms): 226851.1 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 73956.8 | valid tokens/s: 73956.8 | physical tokens/s: 75307.7 | loss tokens/s: 53624.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.229896E-01 | lm-global-supervised-tokens: 1.216468E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.614793E-02 | lm-token-buffer-mib: 2.392523E+02 | global_load_balancing_loss: 1.000278E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11205.296 | params rms: 0.060673 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:18:57.568352 | finish at 2026-10-17 20:00:44 + [2026-10-13 00:45:30] iteration 555/ 2384 | consumed samples: 17760 | elapsed time per iteration (ms): 223458.9 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75079.6 | valid tokens/s: 75079.6 | physical tokens/s: 75855.2 | loss tokens/s: 60095.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.815812E-01 | lm-global-supervised-tokens: 1.342876E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.521943E-02 | lm-token-buffer-mib: 2.343006E+02 | global_load_balancing_loss: 1.000189E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11205.257 | params rms: 0.060673 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:31:46.414395 | finish at 2026-10-17 18:17:16 + [2026-10-13 00:49:14] iteration 556/ 2384 | consumed samples: 17792 | elapsed time per iteration (ms): 224173.6 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 74840.3 | valid tokens/s: 74840.3 | physical tokens/s: 75960.0 | loss tokens/s: 53485.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.924599E-01 | lm-global-supervised-tokens: 1.199001E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.410597E-02 | lm-token-buffer-mib: 2.373657E+02 | global_load_balancing_loss: 1.000669E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11205.219 | params rms: 0.060673 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:49:49.368181 | finish at 2026-10-17 18:39:03 + [2026-10-13 00:52:58] iteration 557/ 2384 | consumed samples: 17824 | elapsed time per iteration (ms): 223877.7 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 74939.0 | valid tokens/s: 74939.0 | physical tokens/s: 75596.7 | loss tokens/s: 56528.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.712556E-01 | lm-global-supervised-tokens: 1.265554E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.503459E-02 | lm-token-buffer-mib: 2.332113E+02 | global_load_balancing_loss: 1.001681E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11205.182 | params rms: 0.060672 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:37:04.590299 | finish at 2026-10-17 18:30:02 + [2026-10-13 00:56:39] iteration 558/ 2384 | consumed samples: 17856 | elapsed time per iteration (ms): 220835.7 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 75971.4 | valid tokens/s: 75971.4 | physical tokens/s: 76636.3 | loss tokens/s: 60658.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.929106E-01 | lm-global-supervised-tokens: 1.339554E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.312152E-02 | lm-token-buffer-mib: 2.330170E+02 | global_load_balancing_loss: 1.000017E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11205.147 | params rms: 0.060672 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:00:46.058356 | finish at 2026-10-17 16:57:25 + [2026-10-13 01:00:20] iteration 559/ 2384 | consumed samples: 17888 | elapsed time per iteration (ms): 221368.9 | throughput per GPU (TFLOP/s/GPU): 27.1 | MFU 8.70% | FLOPs estimator: workload-aware | logical tokens/s: 75788.4 | valid tokens/s: 75788.4 | physical tokens/s: 76048.7 | loss tokens/s: 60297.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.523737E-01 | lm-global-supervised-tokens: 1.334808E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.590396E-02 | lm-token-buffer-mib: 2.332461E+02 | global_load_balancing_loss: 1.000583E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11205.110 | params rms: 0.060672 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:13:18.223031 | finish at 2026-10-17 17:13:38 + [2026-10-13 01:04:29] iteration 560/ 2384 | consumed samples: 17920 | elapsed time per iteration (ms): 249128.1 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.38% | FLOPs estimator: workload-aware | logical tokens/s: 67343.7 | valid tokens/s: 67343.7 | physical tokens/s: 68306.8 | loss tokens/s: 52091.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.545981E-01 | lm-global-supervised-tokens: 1.297757E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.554669E-02 | lm-token-buffer-mib: 2.362462E+02 | global_load_balancing_loss: 1.001030E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11205.072 | params rms: 0.060672 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:13:29.733833 | finish at 2026-10-18 07:17:59 +times across ranks (ms): + save-checkpoint: + rank 0: 25292.67 + rank 1: 25292.69 + rank 2: 25292.69 + rank 3: 25292.66 + rank 4: 25292.65 + rank 5: 25292.66 + rank 6: 25292.67 + rank 7: 25292.67 + rank 8: 25292.87 + rank 9: 25292.92 + rank 10: 25292.92 + rank 11: 25292.87 + rank 12: 25292.90 + rank 13: 25292.87 + rank 14: 25292.88 + rank 15: 25292.88 + rank 16: 25292.63 + rank 17: 25292.67 + rank 18: 25292.69 + rank 19: 25292.67 + rank 20: 25292.63 + rank 21: 25292.63 + rank 22: 25292.70 + rank 23: 25292.62 + rank 24: 25292.64 + rank 25: 25292.67 + rank 26: 25292.62 + rank 27: 25292.61 + rank 28: 25292.63 + rank 29: 25292.69 + rank 30: 25292.39 + rank 31: 25292.66 + rank 32: 25292.85 + rank 33: 25292.71 + rank 34: 25292.86 + rank 35: 25292.85 + rank 36: 25292.80 + rank 37: 25292.84 + rank 38: 25292.82 + rank 39: 25292.99 + rank 40: 25291.78 + rank 41: 25291.77 + rank 42: 25291.77 + rank 43: 25291.80 + rank 44: 25291.78 + rank 45: 25291.77 + rank 46: 25291.76 + rank 47: 25291.80 + rank 48: 25293.21 + rank 49: 25293.12 + rank 50: 25293.19 + rank 51: 25293.16 + rank 52: 25293.03 + rank 53: 25293.16 + rank 54: 25293.14 + rank 55: 25293.09 + rank 56: 25292.36 + rank 57: 25292.40 + rank 58: 25292.38 + rank 59: 25292.39 + rank 60: 25292.40 + rank 61: 25292.38 + rank 62: 25292.36 + rank 63: 25292.36 + [2026-10-13 01:08:42] iteration 561/ 2384 | consumed samples: 17952 | elapsed time per iteration (ms): 227440.9 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 73765.1 | valid tokens/s: 73765.1 | physical tokens/s: 74564.3 | loss tokens/s: 58994.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.665864E-01 | lm-global-supervised-tokens: 1.341785E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.759065E-02 | lm-token-buffer-mib: 2.358089E+02 | global_load_balancing_loss: 9.995201E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11205.034 | params rms: 0.060672 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:10:24.691317 | finish at 2026-10-17 20:19:07 + [2026-10-13 01:12:32] iteration 562/ 2384 | consumed samples: 17984 | elapsed time per iteration (ms): 229709.3 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 73036.5 | valid tokens/s: 73036.5 | physical tokens/s: 73629.9 | loss tokens/s: 56446.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.084627E-01 | lm-global-supervised-tokens: 1.296617E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.631398E-01 | lm-token-buffer-mib: 2.332873E+02 | global_load_balancing_loss: 9.997153E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11204.995 | params rms: 0.060671 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:15:30.276040 | finish at 2026-10-17 21:28:02 + [2026-10-13 01:16:15] iteration 563/ 2384 | consumed samples: 18016 | elapsed time per iteration (ms): 222771.0 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 75311.3 | valid tokens/s: 75311.3 | physical tokens/s: 75801.9 | loss tokens/s: 55090.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.641079E-01 | lm-global-supervised-tokens: 1.227248E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.709765E-02 | lm-token-buffer-mib: 2.336417E+02 | global_load_balancing_loss: 9.999418E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11204.956 | params rms: 0.060671 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:41:05.988228 | finish at 2026-10-17 17:57:20 + [2026-10-13 01:19:56] iteration 564/ 2384 | consumed samples: 18048 | elapsed time per iteration (ms): 221902.3 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.97% | FLOPs estimator: workload-aware | logical tokens/s: 75606.3 | valid tokens/s: 75606.3 | physical tokens/s: 76286.1 | loss tokens/s: 57511.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.898023E-01 | lm-global-supervised-tokens: 1.276183E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.231941E-02 | lm-token-buffer-mib: 2.348663E+02 | global_load_balancing_loss: 1.000415E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11204.916 | params rms: 0.060671 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:11:02.195330 | finish at 2026-10-17 17:30:59 + [2026-10-13 01:23:37] iteration 565/ 2384 | consumed samples: 18080 | elapsed time per iteration (ms): 221054.5 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 75896.3 | valid tokens/s: 75896.3 | physical tokens/s: 76508.1 | loss tokens/s: 58863.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.538867E-01 | lm-global-supervised-tokens: 1.301212E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.253169E-02 | lm-token-buffer-mib: 2.345315E+02 | global_load_balancing_loss: 9.999915E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11204.876 | params rms: 0.060671 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:41:38.107932 | finish at 2026-10-17 17:05:16 + [2026-10-13 01:27:21] iteration 566/ 2384 | consumed samples: 18112 | elapsed time per iteration (ms): 223800.0 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 74965.1 | valid tokens/s: 74965.1 | physical tokens/s: 75447.4 | loss tokens/s: 54050.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.183566E-01 | lm-global-supervised-tokens: 1.209660E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.678100E-02 | lm-token-buffer-mib: 2.336184E+02 | global_load_balancing_loss: 1.001861E+00 | loss scale: 1.0 | grad norm: 0.302 | params norm: 11204.833 | params rms: 0.060671 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:01:08.463630 | finish at 2026-10-17 18:28:30 + [2026-10-13 01:31:05] iteration 567/ 2384 | consumed samples: 18144 | elapsed time per iteration (ms): 223495.3 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 75067.3 | valid tokens/s: 75067.3 | physical tokens/s: 76084.3 | loss tokens/s: 59748.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.294975E-01 | lm-global-supervised-tokens: 1.335356E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.347149E-02 | lm-token-buffer-mib: 2.343434E+02 | global_load_balancing_loss: 1.000733E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11204.784 | params rms: 0.060670 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:48:11.042075 | finish at 2026-10-17 18:19:16 + [2026-10-13 01:34:53] iteration 568/ 2384 | consumed samples: 18176 | elapsed time per iteration (ms): 228103.1 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.96% | FLOPs estimator: workload-aware | logical tokens/s: 73550.9 | valid tokens/s: 73550.9 | physical tokens/s: 74524.6 | loss tokens/s: 54556.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.099010E-01 | lm-global-supervised-tokens: 1.244454E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.618010E-02 | lm-token-buffer-mib: 2.363521E+02 | global_load_balancing_loss: 1.001177E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11204.744 | params rms: 0.060670 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:03:55.306347 | finish at 2026-10-17 20:38:48 + [2026-10-13 01:38:36] iteration 569/ 2384 | consumed samples: 18208 | elapsed time per iteration (ms): 222846.0 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.28% | FLOPs estimator: workload-aware | logical tokens/s: 75286.0 | valid tokens/s: 75286.0 | physical tokens/s: 75843.9 | loss tokens/s: 54352.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.961374E-01 | lm-global-supervised-tokens: 1.211229E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.898095E-02 | lm-token-buffer-mib: 2.341450E+02 | global_load_balancing_loss: 9.992638E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11204.710 | params rms: 0.060670 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:21:05.440589 | finish at 2026-10-17 17:59:41 + [2026-10-13 01:42:39] iteration 570/ 2384 | consumed samples: 18240 | elapsed time per iteration (ms): 243602.1 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 68871.2 | valid tokens/s: 68871.2 | physical tokens/s: 69593.1 | loss tokens/s: 53737.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.816777E-01 | lm-global-supervised-tokens: 1.309053E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.433568E-02 | lm-token-buffer-mib: 2.348718E+02 | global_load_balancing_loss: 9.990055E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11204.677 | params rms: 0.060670 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:44:54.160339 | finish at 2026-10-18 04:27:33 + [2026-10-13 01:46:22] iteration 571/ 2384 | consumed samples: 18272 | elapsed time per iteration (ms): 223040.8 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.30% | FLOPs estimator: workload-aware | logical tokens/s: 75220.0 | valid tokens/s: 75220.0 | physical tokens/s: 75437.6 | loss tokens/s: 56620.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.270092E-01 | lm-global-supervised-tokens: 1.262864E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.493080E-02 | lm-token-buffer-mib: 2.325663E+02 | global_load_balancing_loss: 9.992083E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11204.644 | params rms: 0.060670 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:19:33.037571 | finish at 2026-10-17 18:05:55 + [2026-10-13 01:50:06] iteration 572/ 2384 | consumed samples: 18304 | elapsed time per iteration (ms): 223184.2 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 75172.0 | valid tokens/s: 75172.0 | physical tokens/s: 76302.5 | loss tokens/s: 57416.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.110172E-01 | lm-global-supervised-tokens: 1.281440E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.889959E-02 | lm-token-buffer-mib: 2.345185E+02 | global_load_balancing_loss: 1.002293E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11204.613 | params rms: 0.060669 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:20:09.743703 | finish at 2026-10-17 18:10:15 + [2026-10-13 01:53:50] iteration 573/ 2384 | consumed samples: 18336 | elapsed time per iteration (ms): 224909.0 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 74595.4 | valid tokens/s: 74595.4 | physical tokens/s: 75567.1 | loss tokens/s: 56376.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.287075E-01 | lm-global-supervised-tokens: 1.267953E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.595356E-02 | lm-token-buffer-mib: 2.365737E+02 | global_load_balancing_loss: 1.002040E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11204.581 | params rms: 0.060669 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:08:30.197991 | finish at 2026-10-17 19:02:21 + [2026-10-13 01:57:39] iteration 574/ 2384 | consumed samples: 18368 | elapsed time per iteration (ms): 228190.7 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 73522.6 | valid tokens/s: 73522.6 | physical tokens/s: 73955.8 | loss tokens/s: 53748.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.414078E-01 | lm-global-supervised-tokens: 1.226494E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.210841E-02 | lm-token-buffer-mib: 2.333975E+02 | global_load_balancing_loss: 1.001313E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11204.549 | params rms: 0.060669 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:43:45.239596 | finish at 2026-10-17 20:41:24 + [2026-10-13 02:01:30] iteration 575/ 2384 | consumed samples: 18400 | elapsed time per iteration (ms): 231515.5 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.10% | FLOPs estimator: workload-aware | logical tokens/s: 72466.8 | valid tokens/s: 72466.8 | physical tokens/s: 73569.9 | loss tokens/s: 54781.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.750300E-01 | lm-global-supervised-tokens: 1.268284E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.965396E-01 | lm-token-buffer-mib: 2.346661E+02 | global_load_balancing_loss: 1.000179E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11204.518 | params rms: 0.060669 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:20:11.584911 | finish at 2026-10-17 22:21:42 + [2026-10-13 02:05:20] iteration 576/ 2384 | consumed samples: 18432 | elapsed time per iteration (ms): 229729.8 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 73030.1 | valid tokens/s: 73030.1 | physical tokens/s: 73883.0 | loss tokens/s: 54629.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.301012E-01 | lm-global-supervised-tokens: 1.255012E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.299895E-02 | lm-token-buffer-mib: 2.367236E+02 | global_load_balancing_loss: 1.000232E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11204.488 | params rms: 0.060669 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:22:31.467598 | finish at 2026-10-17 21:27:51 + [2026-10-13 02:09:01] iteration 577/ 2384 | consumed samples: 18464 | elapsed time per iteration (ms): 220638.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 76039.5 | valid tokens/s: 76039.5 | physical tokens/s: 76271.9 | loss tokens/s: 51213.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.130396E-01 | lm-global-supervised-tokens: 1.129963E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.294178E-02 | lm-token-buffer-mib: 2.326840E+02 | global_load_balancing_loss: 9.984412E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11204.454 | params rms: 0.060668 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:44:53.003453 | finish at 2026-10-17 16:53:54 + [2026-10-13 02:12:42] iteration 578/ 2384 | consumed samples: 18496 | elapsed time per iteration (ms): 221625.1 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.53% | FLOPs estimator: workload-aware | logical tokens/s: 75700.8 | valid tokens/s: 75700.8 | physical tokens/s: 76048.3 | loss tokens/s: 58819.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.114124E-01 | lm-global-supervised-tokens: 1.303591E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.501013E-02 | lm-token-buffer-mib: 2.328669E+02 | global_load_balancing_loss: 9.993094E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11204.416 | params rms: 0.060668 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:10:54.869272 | finish at 2026-10-17 17:23:37 + [2026-10-13 02:16:22] iteration 579/ 2384 | consumed samples: 18528 | elapsed time per iteration (ms): 220050.3 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 76242.4 | valid tokens/s: 76242.3 | physical tokens/s: 77013.1 | loss tokens/s: 56171.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.686311E-01 | lm-global-supervised-tokens: 1.236045E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.693961E-02 | lm-token-buffer-mib: 2.337024E+02 | global_load_balancing_loss: 9.994152E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11204.383 | params rms: 0.060668 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:19:50.833462 | finish at 2026-10-17 16:36:13 + [2026-10-13 02:20:25] iteration 580/ 2384 | consumed samples: 18560 | elapsed time per iteration (ms): 242777.6 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 69105.1 | valid tokens/s: 69105.1 | physical tokens/s: 69726.4 | loss tokens/s: 51313.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.829905E-01 | lm-global-supervised-tokens: 1.245771E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.198496E-02 | lm-token-buffer-mib: 2.335604E+02 | global_load_balancing_loss: 9.995284E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11204.350 | params rms: 0.060668 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:39:30.871780 | finish at 2026-10-18 03:59:56 + [2026-10-13 02:24:08] iteration 581/ 2384 | consumed samples: 18592 | elapsed time per iteration (ms): 222530.2 | throughput per GPU (TFLOP/s/GPU): 26.0 | MFU 8.33% | FLOPs estimator: workload-aware | logical tokens/s: 75392.7 | valid tokens/s: 75392.7 | physical tokens/s: 75694.1 | loss tokens/s: 54344.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.689075E-01 | lm-global-supervised-tokens: 1.209338E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.999877E-02 | lm-token-buffer-mib: 2.328524E+02 | global_load_balancing_loss: 1.003002E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11204.312 | params rms: 0.060668 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:27:02.013369 | finish at 2026-10-17 17:51:10 + [2026-10-13 02:27:52] iteration 582/ 2384 | consumed samples: 18624 | elapsed time per iteration (ms): 224968.9 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.88% | FLOPs estimator: workload-aware | logical tokens/s: 74575.6 | valid tokens/s: 74575.6 | physical tokens/s: 75362.6 | loss tokens/s: 56865.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.675546E-01 | lm-global-supervised-tokens: 1.279287E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.737363E-02 | lm-token-buffer-mib: 2.332477E+02 | global_load_balancing_loss: 1.005943E+00 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11204.273 | params rms: 0.060667 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:36:34.042804 | finish at 2026-10-17 19:04:27 + [2026-10-13 02:31:32] iteration 583/ 2384 | consumed samples: 18656 | elapsed time per iteration (ms): 219219.6 | throughput per GPU (TFLOP/s/GPU): 18.4 | MFU 5.91% | FLOPs estimator: workload-aware | logical tokens/s: 76531.5 | valid tokens/s: 76531.5 | physical tokens/s: 77237.2 | loss tokens/s: 61531.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.776514E-01 | lm-global-supervised-tokens: 1.348886E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.795881E-02 | lm-token-buffer-mib: 2.332269E+02 | global_load_balancing_loss: 1.001071E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11204.234 | params rms: 0.060667 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:40:14.470333 | finish at 2026-10-17 16:11:46 + [2026-10-13 02:35:15] iteration 584/ 2384 | consumed samples: 18688 | elapsed time per iteration (ms): 223597.8 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 75032.9 | valid tokens/s: 75032.9 | physical tokens/s: 75636.1 | loss tokens/s: 60449.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.016011E-01 | lm-global-supervised-tokens: 1.351630E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.671587E-02 | lm-token-buffer-mib: 2.339234E+02 | global_load_balancing_loss: 1.001232E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11204.196 | params rms: 0.060667 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:47:56.090240 | finish at 2026-10-17 18:23:11 + [2026-10-13 02:38:59] iteration 585/ 2384 | consumed samples: 18720 | elapsed time per iteration (ms): 223943.4 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 74917.2 | valid tokens/s: 74917.2 | physical tokens/s: 75288.5 | loss tokens/s: 61258.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.135029E-01 | lm-global-supervised-tokens: 1.371844E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.386728E-02 | lm-token-buffer-mib: 2.332994E+02 | global_load_balancing_loss: 1.001688E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11204.154 | params rms: 0.060667 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:54:34.102658 | finish at 2026-10-17 18:33:33 + [2026-10-13 02:42:42] iteration 586/ 2384 | consumed samples: 18752 | elapsed time per iteration (ms): 222964.0 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 75246.1 | valid tokens/s: 75246.1 | physical tokens/s: 75976.5 | loss tokens/s: 59102.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.903259E-01 | lm-global-supervised-tokens: 1.317774E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.457065E-02 | lm-token-buffer-mib: 2.342523E+02 | global_load_balancing_loss: 9.988787E-01 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11204.110 | params rms: 0.060667 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:21:29.356854 | finish at 2026-10-17 18:04:12 + [2026-10-13 02:46:26] iteration 587/ 2384 | consumed samples: 18784 | elapsed time per iteration (ms): 223710.8 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 74995.0 | valid tokens/s: 74995.0 | physical tokens/s: 75740.6 | loss tokens/s: 56832.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.841326E-01 | lm-global-supervised-tokens: 1.271402E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.470576E-02 | lm-token-buffer-mib: 2.344933E+02 | global_load_balancing_loss: 9.974823E-01 | loss scale: 1.0 | grad norm: 0.168 | params norm: 11204.066 | params rms: 0.060666 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:40:08.280915 | finish at 2026-10-17 18:26:34 + [2026-10-13 02:50:08] iteration 588/ 2384 | consumed samples: 18816 | elapsed time per iteration (ms): 222287.4 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 75475.3 | valid tokens/s: 75475.3 | physical tokens/s: 75885.0 | loss tokens/s: 55080.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.617960E-01 | lm-global-supervised-tokens: 1.224364E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.562313E-02 | lm-token-buffer-mib: 2.325891E+02 | global_load_balancing_loss: 9.995146E-01 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11204.023 | params rms: 0.060666 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:53:48.116888 | finish at 2026-10-17 17:43:56 + [2026-10-13 02:53:51] iteration 589/ 2384 | consumed samples: 18848 | elapsed time per iteration (ms): 222282.1 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 75477.0 | valid tokens/s: 75477.0 | physical tokens/s: 76743.4 | loss tokens/s: 59017.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.932295E-01 | lm-global-supervised-tokens: 1.311859E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.335755E-02 | lm-token-buffer-mib: 2.350905E+02 | global_load_balancing_loss: 9.994560E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11203.980 | params rms: 0.060666 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:49:56.365581 | finish at 2026-10-17 17:43:47 + [2026-10-13 02:57:56] iteration 590/ 2384 | consumed samples: 18880 | elapsed time per iteration (ms): 245472.3 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 68346.4 | valid tokens/s: 68346.4 | physical tokens/s: 68459.6 | loss tokens/s: 46005.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.072661E-01 | lm-global-supervised-tokens: 1.129302E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.681165E-02 | lm-token-buffer-mib: 2.322645E+02 | global_load_balancing_loss: 1.002416E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11203.939 | params rms: 0.060666 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:19:37.302017 | finish at 2026-10-18 05:17:33 + [2026-10-13 03:02:30] iteration 591/ 2384 | consumed samples: 18912 | elapsed time per iteration (ms): 273905.0 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.95% | FLOPs estimator: workload-aware | logical tokens/s: 61251.9 | valid tokens/s: 61251.9 | physical tokens/s: 62486.4 | loss tokens/s: 49101.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.880160E-01 | lm-global-supervised-tokens: 1.344909E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.947806E-02 | lm-token-buffer-mib: 2.400619E+02 | global_load_balancing_loss: 1.001467E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11203.893 | params rms: 0.060665 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:25:11.709835 | finish at 2026-10-18 19:27:42 + [2026-10-13 03:06:10] iteration 592/ 2384 | consumed samples: 18944 | elapsed time per iteration (ms): 220251.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 76173.1 | valid tokens/s: 76173.1 | physical tokens/s: 76729.9 | loss tokens/s: 54946.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.292004E-01 | lm-global-supervised-tokens: 1.210199E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.277787E-02 | lm-token-buffer-mib: 2.335594E+02 | global_load_balancing_loss: 1.001464E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11203.848 | params rms: 0.060665 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:38:10.101196 | finish at 2026-10-17 16:44:20 + [2026-10-13 03:09:55] iteration 593/ 2384 | consumed samples: 18976 | elapsed time per iteration (ms): 224380.4 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 74771.3 | valid tokens/s: 74771.3 | physical tokens/s: 75276.5 | loss tokens/s: 58843.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.758725E-01 | lm-global-supervised-tokens: 1.320325E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.637481E-02 | lm-token-buffer-mib: 2.335656E+02 | global_load_balancing_loss: 1.000558E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11203.798 | params rms: 0.060665 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:37:45.227549 | finish at 2026-10-17 18:47:40 + [2026-10-13 03:13:36] iteration 594/ 2384 | consumed samples: 19008 | elapsed time per iteration (ms): 221666.8 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.68% | FLOPs estimator: workload-aware | logical tokens/s: 75686.4 | valid tokens/s: 75686.4 | physical tokens/s: 76138.2 | loss tokens/s: 59130.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.056802E-01 | lm-global-supervised-tokens: 1.310729E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.839070E-02 | lm-token-buffer-mib: 2.340498E+02 | global_load_balancing_loss: 9.984189E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11203.745 | params rms: 0.060665 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:13:03.591244 | finish at 2026-10-17 17:26:40 + [2026-10-13 03:17:22] iteration 595/ 2384 | consumed samples: 19040 | elapsed time per iteration (ms): 225544.8 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 74385.2 | valid tokens/s: 74385.2 | physical tokens/s: 75130.2 | loss tokens/s: 57692.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.343420E-01 | lm-global-supervised-tokens: 1.301219E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.909078E-02 | lm-token-buffer-mib: 2.336053E+02 | global_load_balancing_loss: 1.001197E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11203.691 | params rms: 0.060664 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:04:59.654102 | finish at 2026-10-17 19:22:21 + [2026-10-13 03:21:05] iteration 596/ 2384 | consumed samples: 19072 | elapsed time per iteration (ms): 223039.6 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 75220.7 | valid tokens/s: 75220.7 | physical tokens/s: 76696.8 | loss tokens/s: 53736.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.598952E-01 | lm-global-supervised-tokens: 1.198533E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.499308E-02 | lm-token-buffer-mib: 2.367990E+02 | global_load_balancing_loss: 9.992596E-01 | loss scale: 1.0 | grad norm: 0.357 | params norm: 11203.636 | params rms: 0.060664 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:46:34.811860 | finish at 2026-10-17 18:07:40 + [2026-10-13 03:24:46] iteration 597/ 2384 | consumed samples: 19104 | elapsed time per iteration (ms): 221421.4 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75770.5 | valid tokens/s: 75770.5 | physical tokens/s: 76358.4 | loss tokens/s: 56762.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.370320E-01 | lm-global-supervised-tokens: 1.256836E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.674327E-02 | lm-token-buffer-mib: 2.352552E+02 | global_load_balancing_loss: 9.996377E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11203.583 | params rms: 0.060664 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:54:40.088791 | finish at 2026-10-17 17:19:26 + [2026-10-13 03:28:30] iteration 598/ 2384 | consumed samples: 19136 | elapsed time per iteration (ms): 223934.8 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 74920.0 | valid tokens/s: 74920.0 | physical tokens/s: 75664.9 | loss tokens/s: 58275.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.657025E-01 | lm-global-supervised-tokens: 1.305001E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.443296E-02 | lm-token-buffer-mib: 2.340999E+02 | global_load_balancing_loss: 1.001133E+00 | loss scale: 1.0 | grad norm: 0.226 | params norm: 11203.531 | params rms: 0.060663 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:05:47.614382 | finish at 2026-10-17 18:34:18 + [2026-10-13 03:32:12] iteration 599/ 2384 | consumed samples: 19168 | elapsed time per iteration (ms): 222238.2 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 75491.8 | valid tokens/s: 75491.8 | physical tokens/s: 76110.3 | loss tokens/s: 55493.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.745400E-01 | lm-global-supervised-tokens: 1.233285E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.258851E-02 | lm-token-buffer-mib: 2.336561E+02 | global_load_balancing_loss: 9.995465E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11203.477 | params rms: 0.060663 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:11:35.147331 | finish at 2026-10-17 17:43:48 + [2026-10-13 03:36:16] iteration 600/ 2384 | consumed samples: 19200 | elapsed time per iteration (ms): 243753.8 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 68828.4 | valid tokens/s: 68828.4 | physical tokens/s: 69459.2 | loss tokens/s: 52201.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.055326E-01 | lm-global-supervised-tokens: 1.272436E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.371637E-02 | lm-token-buffer-mib: 2.344358E+02 | global_load_balancing_loss: 1.002605E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11203.425 | params rms: 0.060663 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:47:36.818605 | finish at 2026-10-18 04:23:53 +times across ranks (ms): + save-checkpoint: + rank 0: 25671.15 + rank 1: 25671.18 + rank 2: 25671.21 + rank 3: 25671.18 + rank 4: 25671.18 + rank 5: 25671.20 + rank 6: 25671.21 + rank 7: 25671.18 + rank 8: 25670.93 + rank 9: 25670.98 + rank 10: 25670.97 + rank 11: 25670.94 + rank 12: 25670.98 + rank 13: 25670.93 + rank 14: 25670.93 + rank 15: 25670.93 + rank 16: 25669.48 + rank 17: 25669.48 + rank 18: 25669.50 + rank 19: 25669.51 + rank 20: 25669.46 + rank 21: 25669.45 + rank 22: 25669.49 + rank 23: 25669.46 + rank 24: 25670.44 + rank 25: 25670.47 + rank 26: 25670.40 + rank 27: 25670.38 + rank 28: 25670.15 + rank 29: 25670.45 + rank 30: 25670.47 + rank 31: 25670.46 + rank 32: 25670.60 + rank 33: 25670.51 + rank 34: 25670.66 + rank 35: 25670.56 + rank 36: 25670.51 + rank 37: 25670.72 + rank 38: 25670.56 + rank 39: 25670.49 + rank 40: 25670.84 + rank 41: 25670.87 + rank 42: 25670.88 + rank 43: 25670.86 + rank 44: 25670.84 + rank 45: 25670.84 + rank 46: 25670.83 + rank 47: 25670.83 + rank 48: 25670.79 + rank 49: 25670.76 + rank 50: 25670.91 + rank 51: 25670.79 + rank 52: 25670.80 + rank 53: 25670.78 + rank 54: 25670.87 + rank 55: 25670.79 + rank 56: 25669.99 + rank 57: 25670.02 + rank 58: 25670.02 + rank 59: 25670.01 + rank 60: 25670.02 + rank 61: 25669.98 + rank 62: 25669.99 + rank 63: 25669.99 + [2026-10-13 03:40:21] iteration 601/ 2384 | consumed samples: 19232 | elapsed time per iteration (ms): 219154.7 | throughput per GPU (TFLOP/s/GPU): 26.6 | MFU 8.52% | FLOPs estimator: workload-aware | logical tokens/s: 76554.1 | valid tokens/s: 76554.0 | physical tokens/s: 77122.8 | loss tokens/s: 64341.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.662224E-01 | lm-global-supervised-tokens: 1.410065E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.338111E-01 | lm-token-buffer-mib: 2.341162E+02 | global_load_balancing_loss: 1.005542E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11203.373 | params rms: 0.060663 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:32:32.783412 | finish at 2026-10-17 16:12:54 + [2026-10-13 03:44:05] iteration 602/ 2384 | consumed samples: 19264 | elapsed time per iteration (ms): 224383.1 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 74770.3 | valid tokens/s: 74770.3 | physical tokens/s: 75496.0 | loss tokens/s: 57369.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.388222E-01 | lm-global-supervised-tokens: 1.287275E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.781816E-02 | lm-token-buffer-mib: 2.339387E+02 | global_load_balancing_loss: 1.001450E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11203.320 | params rms: 0.060662 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:04:10.760734 | finish at 2026-10-17 18:48:16 + [2026-10-13 03:47:51] iteration 603/ 2384 | consumed samples: 19296 | elapsed time per iteration (ms): 225821.8 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 74293.9 | valid tokens/s: 74293.9 | physical tokens/s: 75106.0 | loss tokens/s: 52301.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.620785E-01 | lm-global-supervised-tokens: 1.181090E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.609265E-02 | lm-token-buffer-mib: 2.350935E+02 | global_load_balancing_loss: 1.000572E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11203.266 | params rms: 0.060662 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:43:08.677593 | finish at 2026-10-17 19:31:00 + [2026-10-13 03:51:34] iteration 604/ 2384 | consumed samples: 19328 | elapsed time per iteration (ms): 222388.1 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 75441.1 | valid tokens/s: 75441.1 | physical tokens/s: 76359.5 | loss tokens/s: 59699.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.665571E-01 | lm-global-supervised-tokens: 1.327639E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.273328E-02 | lm-token-buffer-mib: 2.354129E+02 | global_load_balancing_loss: 9.996791E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11203.210 | params rms: 0.060662 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:57:30.786009 | finish at 2026-10-17 17:49:04 + [2026-10-13 03:55:15] iteration 605/ 2384 | consumed samples: 19360 | elapsed time per iteration (ms): 221574.6 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.44% | FLOPs estimator: workload-aware | logical tokens/s: 75718.1 | valid tokens/s: 75718.0 | physical tokens/s: 75934.8 | loss tokens/s: 53132.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.238971E-01 | lm-global-supervised-tokens: 1.177290E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.545798E-02 | lm-token-buffer-mib: 2.326534E+02 | global_load_balancing_loss: 9.996078E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11203.157 | params rms: 0.060661 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:29:41.128537 | finish at 2026-10-17 17:24:56 + [2026-10-13 03:58:56] iteration 606/ 2384 | consumed samples: 19392 | elapsed time per iteration (ms): 220836.5 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75971.2 | valid tokens/s: 75971.2 | physical tokens/s: 76649.4 | loss tokens/s: 58233.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.301130E-01 | lm-global-supervised-tokens: 1.286007E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.280066E-02 | lm-token-buffer-mib: 2.334947E+02 | global_load_balancing_loss: 9.993919E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11203.105 | params rms: 0.060661 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:04:07.264657 | finish at 2026-10-17 17:03:03 + [2026-10-13 04:02:38] iteration 607/ 2384 | consumed samples: 19424 | elapsed time per iteration (ms): 221593.2 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.07% | FLOPs estimator: workload-aware | logical tokens/s: 75711.6 | valid tokens/s: 75711.6 | physical tokens/s: 76537.5 | loss tokens/s: 58154.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.392829E-01 | lm-global-supervised-tokens: 1.288656E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.410982E-02 | lm-token-buffer-mib: 2.353925E+02 | global_load_balancing_loss: 9.999895E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11203.053 | params rms: 0.060661 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:22:51.041354 | finish at 2026-10-17 17:25:29 + [2026-10-13 04:06:19] iteration 608/ 2384 | consumed samples: 19456 | elapsed time per iteration (ms): 221670.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75685.2 | valid tokens/s: 75685.2 | physical tokens/s: 76354.1 | loss tokens/s: 59363.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.534832E-01 | lm-global-supervised-tokens: 1.315914E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.679457E-02 | lm-token-buffer-mib: 2.345078E+02 | global_load_balancing_loss: 1.000132E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11203.003 | params rms: 0.060661 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:21:26.788593 | finish at 2026-10-17 17:27:46 + [2026-10-13 04:10:03] iteration 609/ 2384 | consumed samples: 19488 | elapsed time per iteration (ms): 223507.2 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 75063.4 | valid tokens/s: 75063.4 | physical tokens/s: 76149.8 | loss tokens/s: 58674.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.320322E-01 | lm-global-supervised-tokens: 1.311425E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.790782E-02 | lm-token-buffer-mib: 2.366143E+02 | global_load_balancing_loss: 1.000745E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11202.954 | params rms: 0.060660 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:12:05.326556 | finish at 2026-10-17 18:22:08 + [2026-10-13 04:14:09] iteration 610/ 2384 | consumed samples: 19520 | elapsed time per iteration (ms): 245948.7 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 68214.3 | valid tokens/s: 68214.3 | physical tokens/s: 68659.3 | loss tokens/s: 54130.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.395326E-01 | lm-global-supervised-tokens: 1.331319E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.516113E-02 | lm-token-buffer-mib: 2.337636E+02 | global_load_balancing_loss: 1.000923E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11202.904 | params rms: 0.060660 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:11:52.910814 | finish at 2026-10-18 05:26:02 + [2026-10-13 04:17:53] iteration 611/ 2384 | consumed samples: 19552 | elapsed time per iteration (ms): 224461.0 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 74744.1 | valid tokens/s: 74744.1 | physical tokens/s: 76110.5 | loss tokens/s: 59343.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.661605E-01 | lm-global-supervised-tokens: 1.332024E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.696688E-02 | lm-token-buffer-mib: 2.340556E+02 | global_load_balancing_loss: 1.001353E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11202.857 | params rms: 0.060660 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:32:49.385911 | finish at 2026-10-17 18:50:43 + [2026-10-13 04:21:37] iteration 612/ 2384 | consumed samples: 19584 | elapsed time per iteration (ms): 223877.4 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 74939.3 | valid tokens/s: 74939.3 | physical tokens/s: 75637.1 | loss tokens/s: 57278.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.734189E-01 | lm-global-supervised-tokens: 1.282344E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.342267E-02 | lm-token-buffer-mib: 2.338633E+02 | global_load_balancing_loss: 1.002135E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11202.811 | params rms: 0.060660 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:11:50.791529 | finish at 2026-10-17 18:33:28 + [2026-10-13 04:25:19] iteration 613/ 2384 | consumed samples: 19616 | elapsed time per iteration (ms): 221893.9 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.28% | FLOPs estimator: workload-aware | logical tokens/s: 75609.0 | valid tokens/s: 75609.0 | physical tokens/s: 75910.3 | loss tokens/s: 56413.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.338480E-01 | lm-global-supervised-tokens: 1.251788E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.999231E-02 | lm-token-buffer-mib: 2.331407E+02 | global_load_balancing_loss: 1.004156E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11202.761 | params rms: 0.060659 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:09:34.136445 | finish at 2026-10-17 17:34:53 + [2026-10-13 04:29:01] iteration 614/ 2384 | consumed samples: 19648 | elapsed time per iteration (ms): 222040.1 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 75559.4 | valid tokens/s: 75559.4 | physical tokens/s: 76248.3 | loss tokens/s: 57300.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.420022E-01 | lm-global-supervised-tokens: 1.272302E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.335292E-02 | lm-token-buffer-mib: 2.344160E+02 | global_load_balancing_loss: 1.000625E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11202.712 | params rms: 0.060659 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:10:11.000383 | finish at 2026-10-17 17:39:12 + [2026-10-13 04:32:45] iteration 615/ 2384 | consumed samples: 19680 | elapsed time per iteration (ms): 223680.7 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 75005.1 | valid tokens/s: 75005.1 | physical tokens/s: 75785.8 | loss tokens/s: 57235.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.740895E-01 | lm-global-supervised-tokens: 1.280246E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.208005E-02 | lm-token-buffer-mib: 2.334775E+02 | global_load_balancing_loss: 9.996907E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11202.668 | params rms: 0.060659 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:54:51.096414 | finish at 2026-10-17 18:27:36 + [2026-10-13 04:36:28] iteration 616/ 2384 | consumed samples: 19712 | elapsed time per iteration (ms): 223020.8 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75227.1 | valid tokens/s: 75227.1 | physical tokens/s: 75505.5 | loss tokens/s: 52763.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.175991E-01 | lm-global-supervised-tokens: 1.176729E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.587383E-02 | lm-token-buffer-mib: 2.327155E+02 | global_load_balancing_loss: 9.962447E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11202.625 | params rms: 0.060659 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:31:40.799892 | finish at 2026-10-17 18:08:09 + [2026-10-13 04:40:10] iteration 617/ 2384 | consumed samples: 19744 | elapsed time per iteration (ms): 222441.6 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75423.0 | valid tokens/s: 75423.0 | physical tokens/s: 76260.9 | loss tokens/s: 56848.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.027050E-01 | lm-global-supervised-tokens: 1.264550E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.112820E-02 | lm-token-buffer-mib: 2.327810E+02 | global_load_balancing_loss: 9.980180E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11202.581 | params rms: 0.060658 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:10:54.245420 | finish at 2026-10-17 17:51:04 + [2026-10-13 04:43:50] iteration 618/ 2384 | consumed samples: 19776 | elapsed time per iteration (ms): 219489.9 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 76437.3 | valid tokens/s: 76437.3 | physical tokens/s: 76937.1 | loss tokens/s: 56614.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.134213E-01 | lm-global-supervised-tokens: 1.242628E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.345174E-02 | lm-token-buffer-mib: 2.330936E+02 | global_load_balancing_loss: 1.000447E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11202.536 | params rms: 0.060658 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:40:19.145914 | finish at 2026-10-17 16:24:09 + [2026-10-13 04:47:29] iteration 619/ 2384 | consumed samples: 19808 | elapsed time per iteration (ms): 219653.1 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 76380.3 | valid tokens/s: 76380.3 | physical tokens/s: 77283.5 | loss tokens/s: 56444.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.453431E-01 | lm-global-supervised-tokens: 1.239822E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.813039E-02 | lm-token-buffer-mib: 2.328650E+02 | global_load_balancing_loss: 1.002038E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11202.489 | params rms: 0.060658 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:41:27.673973 | finish at 2026-10-17 16:28:57 + [2026-10-13 04:51:33] iteration 620/ 2384 | consumed samples: 19840 | elapsed time per iteration (ms): 243439.0 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.29% | FLOPs estimator: workload-aware | logical tokens/s: 68917.3 | valid tokens/s: 68917.3 | physical tokens/s: 69438.1 | loss tokens/s: 51761.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.711902E-01 | lm-global-supervised-tokens: 1.260078E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.528551E-02 | lm-token-buffer-mib: 2.334406E+02 | global_load_balancing_loss: 1.001708E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11202.441 | params rms: 0.060658 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:17:06.392444 | finish at 2026-10-18 04:08:39 + [2026-10-13 04:55:15] iteration 621/ 2384 | consumed samples: 19872 | elapsed time per iteration (ms): 222335.8 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 75458.7 | valid tokens/s: 75458.7 | physical tokens/s: 76529.7 | loss tokens/s: 55656.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.108674E-01 | lm-global-supervised-tokens: 1.237445E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.966786E-02 | lm-token-buffer-mib: 2.349687E+02 | global_load_balancing_loss: 1.002823E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11202.397 | params rms: 0.060657 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:52:58.085476 | finish at 2026-10-17 17:48:13 + [2026-10-13 04:58:57] iteration 622/ 2384 | consumed samples: 19904 | elapsed time per iteration (ms): 221439.3 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75764.3 | valid tokens/s: 75764.3 | physical tokens/s: 76531.7 | loss tokens/s: 57185.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.990704E-01 | lm-global-supervised-tokens: 1.266311E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.572749E-02 | lm-token-buffer-mib: 2.335959E+02 | global_load_balancing_loss: 1.001548E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11202.354 | params rms: 0.060657 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:22:56.039145 | finish at 2026-10-17 17:21:53 + [2026-10-13 05:02:39] iteration 623/ 2384 | consumed samples: 19936 | elapsed time per iteration (ms): 221992.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75575.5 | valid tokens/s: 75575.5 | physical tokens/s: 76106.5 | loss tokens/s: 54223.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.844882E-01 | lm-global-supervised-tokens: 1.203726E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.737830E-02 | lm-token-buffer-mib: 2.334669E+02 | global_load_balancing_loss: 1.000390E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11202.309 | params rms: 0.060657 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:35:27.973060 | finish at 2026-10-17 17:38:06 + [2026-10-13 05:06:21] iteration 624/ 2384 | consumed samples: 19968 | elapsed time per iteration (ms): 222044.0 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.48% | FLOPs estimator: workload-aware | logical tokens/s: 75558.0 | valid tokens/s: 75558.0 | physical tokens/s: 75865.3 | loss tokens/s: 56156.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.201777E-01 | lm-global-supervised-tokens: 1.246927E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.275910E-02 | lm-token-buffer-mib: 2.325933E+02 | global_load_balancing_loss: 1.001470E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11202.263 | params rms: 0.060657 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:33:17.479706 | finish at 2026-10-17 17:39:38 + [2026-10-13 05:10:04] iteration 625/ 2384 | consumed samples: 20000 | elapsed time per iteration (ms): 223358.8 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 75113.1 | valid tokens/s: 75113.1 | physical tokens/s: 75846.2 | loss tokens/s: 58608.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.082367E-01 | lm-global-supervised-tokens: 1.309063E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.422829E-02 | lm-token-buffer-mib: 2.347194E+02 | global_load_balancing_loss: 1.000788E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11202.215 | params rms: 0.060656 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:08:08.196604 | finish at 2026-10-17 18:18:12 + [2026-10-13 05:13:50] iteration 626/ 2384 | consumed samples: 20032 | elapsed time per iteration (ms): 225733.4 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 74323.1 | valid tokens/s: 74323.1 | physical tokens/s: 75086.7 | loss tokens/s: 56723.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.743099E-01 | lm-global-supervised-tokens: 1.280436E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.471409E-02 | lm-token-buffer-mib: 2.337211E+02 | global_load_balancing_loss: 9.989275E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11202.168 | params rms: 0.060656 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:13:59.385707 | finish at 2026-10-17 19:27:49 + [2026-10-13 05:17:33] iteration 627/ 2384 | consumed samples: 20064 | elapsed time per iteration (ms): 223241.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75152.8 | valid tokens/s: 75152.8 | physical tokens/s: 75678.4 | loss tokens/s: 55021.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.416176E-01 | lm-global-supervised-tokens: 1.228297E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.081004E-02 | lm-token-buffer-mib: 2.346298E+02 | global_load_balancing_loss: 1.000740E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11202.122 | params rms: 0.060656 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:57:14.668123 | finish at 2026-10-17 18:14:48 + [2026-10-13 05:21:12] iteration 628/ 2384 | consumed samples: 20096 | elapsed time per iteration (ms): 219262.3 | throughput per GPU (TFLOP/s/GPU): 18.2 | MFU 5.84% | FLOPs estimator: workload-aware | logical tokens/s: 76516.6 | valid tokens/s: 76516.6 | physical tokens/s: 77174.6 | loss tokens/s: 58356.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.854762E-01 | lm-global-supervised-tokens: 1.279542E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.430358E-02 | lm-token-buffer-mib: 2.331735E+02 | global_load_balancing_loss: 1.001534E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11202.077 | params rms: 0.060656 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:57:04.523466 | finish at 2026-10-17 16:18:17 + [2026-10-13 05:24:55] iteration 629/ 2384 | consumed samples: 20128 | elapsed time per iteration (ms): 222708.1 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.91% | FLOPs estimator: workload-aware | logical tokens/s: 75332.6 | valid tokens/s: 75332.6 | physical tokens/s: 75734.8 | loss tokens/s: 55364.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.023188E-01 | lm-global-supervised-tokens: 1.233003E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.477985E-02 | lm-token-buffer-mib: 2.327313E+02 | global_load_balancing_loss: 1.000183E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11202.032 | params rms: 0.060655 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:34:12.639488 | finish at 2026-10-17 17:59:08 + [2026-10-13 05:29:01] iteration 630/ 2384 | consumed samples: 20160 | elapsed time per iteration (ms): 246434.0 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 68079.9 | valid tokens/s: 68079.9 | physical tokens/s: 68791.8 | loss tokens/s: 50427.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.231961E-01 | lm-global-supervised-tokens: 1.242708E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.795591E-02 | lm-token-buffer-mib: 2.334043E+02 | global_load_balancing_loss: 1.000531E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11201.991 | params rms: 0.060655 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:04:05.270318 | finish at 2026-10-18 05:33:07 + [2026-10-13 05:32:45] iteration 631/ 2384 | consumed samples: 20192 | elapsed time per iteration (ms): 223474.7 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.91% | FLOPs estimator: workload-aware | logical tokens/s: 75074.2 | valid tokens/s: 75074.2 | physical tokens/s: 76031.7 | loss tokens/s: 59334.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.527460E-01 | lm-global-supervised-tokens: 1.325982E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.591830E-02 | lm-token-buffer-mib: 2.354978E+02 | global_load_balancing_loss: 9.997386E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11201.951 | params rms: 0.060655 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:49:11.095139 | finish at 2026-10-17 18:21:56 + [2026-10-13 05:36:26] iteration 632/ 2384 | consumed samples: 20224 | elapsed time per iteration (ms): 221611.5 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 75705.3 | valid tokens/s: 75705.3 | physical tokens/s: 76369.2 | loss tokens/s: 54888.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.816506E-01 | lm-global-supervised-tokens: 1.216393E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.065815E-02 | lm-token-buffer-mib: 2.342267E+02 | global_load_balancing_loss: 1.000625E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11201.912 | params rms: 0.060655 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:51:03.285805 | finish at 2026-10-17 17:27:30 + [2026-10-13 05:40:08] iteration 633/ 2384 | consumed samples: 20256 | elapsed time per iteration (ms): 221271.6 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 75821.6 | valid tokens/s: 75821.6 | physical tokens/s: 76551.0 | loss tokens/s: 56159.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.747613E-01 | lm-global-supervised-tokens: 1.242645E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.423437E-02 | lm-token-buffer-mib: 2.333101E+02 | global_load_balancing_loss: 9.995530E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11201.874 | params rms: 0.060655 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:37:26.602924 | finish at 2026-10-17 17:17:34 + [2026-10-13 05:44:01] iteration 634/ 2384 | consumed samples: 20288 | elapsed time per iteration (ms): 233295.0 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 71914.0 | valid tokens/s: 71914.0 | physical tokens/s: 73075.4 | loss tokens/s: 51947.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.511941E-01 | lm-global-supervised-tokens: 1.211915E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.257440E-02 | lm-token-buffer-mib: 2.389834E+02 | global_load_balancing_loss: 9.997566E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11201.833 | params rms: 0.060654 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:24:26.254306 | finish at 2026-10-17 23:08:27 + [2026-10-13 05:47:43] iteration 635/ 2384 | consumed samples: 20320 | elapsed time per iteration (ms): 221623.4 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 75701.3 | valid tokens/s: 75701.3 | physical tokens/s: 76365.7 | loss tokens/s: 58815.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.381399E-01 | lm-global-supervised-tokens: 1.303480E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.214845E-02 | lm-token-buffer-mib: 2.325991E+02 | global_load_balancing_loss: 1.000428E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11201.791 | params rms: 0.060654 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:40:19.296946 | finish at 2026-10-17 17:28:02 + [2026-10-13 05:51:29] iteration 636/ 2384 | consumed samples: 20352 | elapsed time per iteration (ms): 225996.1 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.97% | FLOPs estimator: workload-aware | logical tokens/s: 74236.7 | valid tokens/s: 74236.7 | physical tokens/s: 75736.2 | loss tokens/s: 56442.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.027473E-01 | lm-global-supervised-tokens: 1.275580E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.752362E-02 | lm-token-buffer-mib: 2.384713E+02 | global_load_balancing_loss: 1.000821E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11201.753 | params rms: 0.060654 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:44:01.205215 | finish at 2026-10-17 19:35:30 + [2026-10-13 05:55:09] iteration 637/ 2384 | consumed samples: 20384 | elapsed time per iteration (ms): 220526.6 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.48% | FLOPs estimator: workload-aware | logical tokens/s: 76077.9 | valid tokens/s: 76077.9 | physical tokens/s: 76630.5 | loss tokens/s: 57676.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.405331E-01 | lm-global-supervised-tokens: 1.271910E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.095434E-02 | lm-token-buffer-mib: 2.338465E+02 | global_load_balancing_loss: 1.000030E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11201.717 | params rms: 0.060654 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:00:59.927929 | finish at 2026-10-17 16:56:09 + [2026-10-13 05:58:51] iteration 638/ 2384 | consumed samples: 20416 | elapsed time per iteration (ms): 221801.6 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 75640.6 | valid tokens/s: 75640.6 | physical tokens/s: 76701.3 | loss tokens/s: 61399.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.265190E-01 | lm-global-supervised-tokens: 1.361853E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.812198E-02 | lm-token-buffer-mib: 2.360113E+02 | global_load_balancing_loss: 1.000648E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11201.678 | params rms: 0.060653 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:34:25.646015 | finish at 2026-10-17 17:33:17 + [2026-10-13 06:02:36] iteration 639/ 2384 | consumed samples: 20448 | elapsed time per iteration (ms): 224756.9 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 74645.7 | valid tokens/s: 74645.7 | physical tokens/s: 75553.0 | loss tokens/s: 52332.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.058490E-01 | lm-global-supervised-tokens: 1.176200E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.507937E-02 | lm-token-buffer-mib: 2.345776E+02 | global_load_balancing_loss: 1.000437E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11201.638 | params rms: 0.060653 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:56:40.850952 | finish at 2026-10-17 18:59:17 + [2026-10-13 06:06:40] iteration 640/ 2384 | consumed samples: 20480 | elapsed time per iteration (ms): 244017.2 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 68754.1 | valid tokens/s: 68754.1 | physical tokens/s: 69091.0 | loss tokens/s: 49115.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.298720E-01 | lm-global-supervised-tokens: 1.198509E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.399739E-02 | lm-token-buffer-mib: 2.333215E+02 | global_load_balancing_loss: 1.001011E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11201.598 | params rms: 0.060653 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:12:46.017162 | finish at 2026-10-18 04:19:26 +times across ranks (ms): + save-checkpoint: + rank 0: 24729.34 + rank 1: 24729.37 + rank 2: 24729.37 + rank 3: 24729.38 + rank 4: 24729.41 + rank 5: 24729.39 + rank 6: 24729.41 + rank 7: 24729.39 + rank 8: 24730.63 + rank 9: 24730.60 + rank 10: 24730.65 + rank 11: 24730.61 + rank 12: 24730.61 + rank 13: 24730.61 + rank 14: 24730.61 + rank 15: 24730.61 + rank 16: 24730.94 + rank 17: 24730.92 + rank 18: 24730.92 + rank 19: 24730.92 + rank 20: 24730.88 + rank 21: 24730.88 + rank 22: 24730.90 + rank 23: 24730.88 + rank 24: 24731.61 + rank 25: 24731.61 + rank 26: 24731.61 + rank 27: 24731.62 + rank 28: 24731.59 + rank 29: 24731.36 + rank 30: 24731.58 + rank 31: 24731.63 + rank 32: 24730.51 + rank 33: 24730.53 + rank 34: 24730.49 + rank 35: 24730.51 + rank 36: 24730.51 + rank 37: 24730.52 + rank 38: 24730.51 + rank 39: 24730.51 + rank 40: 24730.68 + rank 41: 24730.65 + rank 42: 24730.65 + rank 43: 24730.66 + rank 44: 24730.69 + rank 45: 24730.71 + rank 46: 24730.63 + rank 47: 24730.63 + rank 48: 24730.81 + rank 49: 24731.07 + rank 50: 24730.86 + rank 51: 24730.82 + rank 52: 24730.89 + rank 53: 24730.94 + rank 54: 24730.91 + rank 55: 24730.81 + rank 56: 24730.58 + rank 57: 24730.61 + rank 58: 24730.61 + rank 59: 24730.60 + rank 60: 24730.67 + rank 61: 24730.57 + rank 62: 24730.58 + rank 63: 24730.58 + [2026-10-13 06:10:50] iteration 641/ 2384 | consumed samples: 20512 | elapsed time per iteration (ms): 225309.0 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 74463.0 | valid tokens/s: 74463.0 | physical tokens/s: 75494.3 | loss tokens/s: 56938.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.254026E-01 | lm-global-supervised-tokens: 1.282870E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.796557E-02 | lm-token-buffer-mib: 2.375279E+02 | global_load_balancing_loss: 1.001028E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11201.559 | params rms: 0.060653 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:05:13.533419 | finish at 2026-10-17 19:16:03 + [2026-10-13 06:14:38] iteration 642/ 2384 | consumed samples: 20544 | elapsed time per iteration (ms): 228321.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 73480.8 | valid tokens/s: 73480.8 | physical tokens/s: 73972.4 | loss tokens/s: 57088.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.304783E-01 | lm-global-supervised-tokens: 1.303459E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.394258E-02 | lm-token-buffer-mib: 2.334234E+02 | global_load_balancing_loss: 1.000296E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11201.521 | params rms: 0.060653 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:28:55.377376 | finish at 2026-10-17 20:43:33 + [2026-10-13 06:18:18] iteration 643/ 2384 | consumed samples: 20576 | elapsed time per iteration (ms): 219960.7 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 76273.6 | valid tokens/s: 76273.6 | physical tokens/s: 77024.9 | loss tokens/s: 54667.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.797281E-01 | lm-global-supervised-tokens: 1.202477E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.099049E-02 | lm-token-buffer-mib: 2.343063E+02 | global_load_balancing_loss: 9.998002E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11201.483 | params rms: 0.060652 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:22:31.655967 | finish at 2026-10-17 16:40:50 + [2026-10-13 06:22:03] iteration 644/ 2384 | consumed samples: 20608 | elapsed time per iteration (ms): 225255.6 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 74480.7 | valid tokens/s: 74480.7 | physical tokens/s: 75867.0 | loss tokens/s: 54125.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.501955E-01 | lm-global-supervised-tokens: 1.219197E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.639647E-02 | lm-token-buffer-mib: 2.370901E+02 | global_load_balancing_loss: 1.000727E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11201.444 | params rms: 0.060652 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:52:24.725289 | finish at 2026-10-17 19:14:28 + [2026-10-13 06:25:47] iteration 645/ 2384 | consumed samples: 20640 | elapsed time per iteration (ms): 223594.6 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.53% | FLOPs estimator: workload-aware | logical tokens/s: 75034.1 | valid tokens/s: 75034.1 | physical tokens/s: 76010.1 | loss tokens/s: 55973.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.736365E-01 | lm-global-supervised-tokens: 1.251526E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.423732E-02 | lm-token-buffer-mib: 2.342921E+02 | global_load_balancing_loss: 1.002832E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11201.406 | params rms: 0.060652 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:00:31.051625 | finish at 2026-10-17 18:26:18 + [2026-10-13 06:29:27] iteration 646/ 2384 | consumed samples: 20672 | elapsed time per iteration (ms): 220081.1 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.45% | FLOPs estimator: workload-aware | logical tokens/s: 76231.9 | valid tokens/s: 76231.9 | physical tokens/s: 77250.9 | loss tokens/s: 57895.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.033999E-01 | lm-global-supervised-tokens: 1.274176E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.576268E-02 | lm-token-buffer-mib: 2.327570E+02 | global_load_balancing_loss: 1.000086E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11201.373 | params rms: 0.060652 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:15:00.943076 | finish at 2026-10-17 16:44:28 + [2026-10-13 06:33:06] iteration 647/ 2384 | consumed samples: 20704 | elapsed time per iteration (ms): 219111.2 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 6.00% | FLOPs estimator: workload-aware | logical tokens/s: 76569.4 | valid tokens/s: 76569.4 | physical tokens/s: 77165.0 | loss tokens/s: 55125.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.742067E-01 | lm-global-supervised-tokens: 1.207851E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.515361E-02 | lm-token-buffer-mib: 2.331495E+02 | global_load_balancing_loss: 9.993759E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11201.339 | params rms: 0.060652 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:43:16.078777 | finish at 2026-10-17 16:16:22 + [2026-10-13 06:36:50] iteration 648/ 2384 | consumed samples: 20736 | elapsed time per iteration (ms): 223472.4 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.70% | FLOPs estimator: workload-aware | logical tokens/s: 75075.0 | valid tokens/s: 75075.0 | physical tokens/s: 75986.1 | loss tokens/s: 56628.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.401956E-01 | lm-global-supervised-tokens: 1.265492E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.348166E-02 | lm-token-buffer-mib: 2.344633E+02 | global_load_balancing_loss: 9.986839E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11201.300 | params rms: 0.060651 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:45:48.090866 | finish at 2026-10-17 18:22:38 + [2026-10-13 06:40:35] iteration 649/ 2384 | consumed samples: 20768 | elapsed time per iteration (ms): 225760.1 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 74314.2 | valid tokens/s: 74314.2 | physical tokens/s: 74981.4 | loss tokens/s: 49600.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.537059E-01 | lm-global-supervised-tokens: 1.119770E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.601054E-02 | lm-token-buffer-mib: 2.347198E+02 | global_load_balancing_loss: 9.999738E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11201.259 | params rms: 0.060651 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:48:13.800607 | finish at 2026-10-17 19:28:49 + [2026-10-13 06:44:37] iteration 650/ 2384 | consumed samples: 20800 | elapsed time per iteration (ms): 241545.4 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 69457.5 | valid tokens/s: 69457.5 | physical tokens/s: 69970.8 | loss tokens/s: 52643.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.067364E-01 | lm-global-supervised-tokens: 1.271589E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.986692E-02 | lm-token-buffer-mib: 2.332457E+02 | global_load_balancing_loss: 9.999787E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11201.217 | params rms: 0.060651 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:20:39.807358 | finish at 2026-10-18 03:05:17 + [2026-10-13 06:48:15] iteration 651/ 2384 | consumed samples: 20832 | elapsed time per iteration (ms): 218098.8 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 76924.7 | valid tokens/s: 76924.7 | physical tokens/s: 77328.0 | loss tokens/s: 58779.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.364879E-01 | lm-global-supervised-tokens: 1.281966E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.597040E-02 | lm-token-buffer-mib: 2.334258E+02 | global_load_balancing_loss: 1.002494E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11201.177 | params rms: 0.060651 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:59:25.168655 | finish at 2026-10-17 15:47:40 + [2026-10-13 06:52:00] iteration 652/ 2384 | consumed samples: 20864 | elapsed time per iteration (ms): 224886.1 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 74603.1 | valid tokens/s: 74603.1 | physical tokens/s: 75360.7 | loss tokens/s: 54647.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.460218E-01 | lm-global-supervised-tokens: 1.228952E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.741330E-02 | lm-token-buffer-mib: 2.335074E+02 | global_load_balancing_loss: 1.002096E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11201.134 | params rms: 0.060650 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:11:42.776498 | finish at 2026-10-17 19:03:43 + [2026-10-13 06:55:45] iteration 653/ 2384 | consumed samples: 20896 | elapsed time per iteration (ms): 225289.7 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 74469.3 | valid tokens/s: 74469.3 | physical tokens/s: 75072.9 | loss tokens/s: 55893.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.421369E-01 | lm-global-supervised-tokens: 1.259227E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.707870E-02 | lm-token-buffer-mib: 2.341161E+02 | global_load_balancing_loss: 1.001837E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11201.095 | params rms: 0.060650 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:19:36.386563 | finish at 2026-10-17 19:15:22 + [2026-10-13 06:59:27] iteration 654/ 2384 | consumed samples: 20928 | elapsed time per iteration (ms): 222078.2 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 75546.4 | valid tokens/s: 75546.4 | physical tokens/s: 76627.4 | loss tokens/s: 54447.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.186051E-01 | lm-global-supervised-tokens: 1.209159E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.558021E-02 | lm-token-buffer-mib: 2.348950E+02 | global_load_balancing_loss: 1.000748E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11201.058 | params rms: 0.060650 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:43:15.249879 | finish at 2026-10-17 17:42:43 + [2026-10-13 07:03:09] iteration 655/ 2384 | consumed samples: 20960 | elapsed time per iteration (ms): 221805.6 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.53% | FLOPs estimator: workload-aware | logical tokens/s: 75639.1 | valid tokens/s: 75639.1 | physical tokens/s: 76072.1 | loss tokens/s: 56368.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.556603E-01 | lm-global-supervised-tokens: 1.250286E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.624434E-02 | lm-token-buffer-mib: 2.329162E+02 | global_load_balancing_loss: 9.991020E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11201.016 | params rms: 0.060650 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:31:41.900826 | finish at 2026-10-17 17:34:51 + [2026-10-13 07:06:58] iteration 656/ 2384 | consumed samples: 20992 | elapsed time per iteration (ms): 229074.8 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 73238.9 | valid tokens/s: 73238.9 | physical tokens/s: 74753.8 | loss tokens/s: 58967.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.983770E-01 | lm-global-supervised-tokens: 1.350802E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.692005E-02 | lm-token-buffer-mib: 2.386452E+02 | global_load_balancing_loss: 9.992747E-01 | loss scale: 1.0 | grad norm: 0.400 | params norm: 11200.972 | params rms: 0.060650 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:57:21.284500 | finish at 2026-10-17 21:04:19 + [2026-10-13 07:10:40] iteration 657/ 2384 | consumed samples: 21024 | elapsed time per iteration (ms): 221376.9 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.29% | FLOPs estimator: workload-aware | logical tokens/s: 75785.6 | valid tokens/s: 75785.6 | physical tokens/s: 76253.5 | loss tokens/s: 57014.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.288342E-01 | lm-global-supervised-tokens: 1.262160E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.457123E-02 | lm-token-buffer-mib: 2.332248E+02 | global_load_balancing_loss: 9.994736E-01 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11200.929 | params rms: 0.060649 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:11:57.932167 | finish at 2026-10-17 17:22:37 + [2026-10-13 07:14:22] iteration 658/ 2384 | consumed samples: 21056 | elapsed time per iteration (ms): 222369.9 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 75447.0 | valid tokens/s: 75447.0 | physical tokens/s: 76275.1 | loss tokens/s: 55668.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.515880E-01 | lm-global-supervised-tokens: 1.237894E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.701279E-02 | lm-token-buffer-mib: 2.348268E+02 | global_load_balancing_loss: 1.001500E+00 | loss scale: 1.0 | grad norm: 0.316 | params norm: 11200.889 | params rms: 0.060649 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:36:50.419397 | finish at 2026-10-17 17:51:12 + [2026-10-13 07:18:07] iteration 659/ 2384 | consumed samples: 21088 | elapsed time per iteration (ms): 224876.0 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 74606.4 | valid tokens/s: 74606.4 | physical tokens/s: 75242.9 | loss tokens/s: 55761.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.831228E-01 | lm-global-supervised-tokens: 1.253946E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.809410E-02 | lm-token-buffer-mib: 2.356993E+02 | global_load_balancing_loss: 1.001690E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11200.850 | params rms: 0.060649 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:45:11.074787 | finish at 2026-10-17 19:03:18 + [2026-10-13 07:22:14] iteration 660/ 2384 | consumed samples: 21120 | elapsed time per iteration (ms): 247633.9 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.26% | FLOPs estimator: workload-aware | logical tokens/s: 67750.0 | valid tokens/s: 67750.0 | physical tokens/s: 68331.8 | loss tokens/s: 52468.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.009838E-01 | lm-global-supervised-tokens: 1.299299E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.213079E-02 | lm-token-buffer-mib: 2.353724E+02 | global_load_balancing_loss: 1.003753E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11200.810 | params rms: 0.060649 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:35:20.822923 | finish at 2026-10-18 05:57:35 + [2026-10-13 07:25:58] iteration 661/ 2384 | consumed samples: 21152 | elapsed time per iteration (ms): 223820.1 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 74958.4 | valid tokens/s: 74958.4 | physical tokens/s: 75821.2 | loss tokens/s: 53416.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.211795E-01 | lm-global-supervised-tokens: 1.195564E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.466135E-02 | lm-token-buffer-mib: 2.350478E+02 | global_load_balancing_loss: 9.990667E-01 | loss scale: 1.0 | grad norm: 0.312 | params norm: 11200.774 | params rms: 0.060649 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:07:22.081304 | finish at 2026-10-17 18:33:20 + [2026-10-13 07:29:40] iteration 662/ 2384 | consumed samples: 21184 | elapsed time per iteration (ms): 222232.7 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.49% | FLOPs estimator: workload-aware | logical tokens/s: 75493.7 | valid tokens/s: 75493.7 | physical tokens/s: 76135.0 | loss tokens/s: 58946.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.713885E-01 | lm-global-supervised-tokens: 1.309994E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.886215E-02 | lm-token-buffer-mib: 2.333672E+02 | global_load_balancing_loss: 9.988997E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11200.739 | params rms: 0.060648 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:18:04.760498 | finish at 2026-10-17 17:47:45 + [2026-10-13 07:33:22] iteration 663/ 2384 | consumed samples: 21216 | elapsed time per iteration (ms): 221681.0 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.96% | FLOPs estimator: workload-aware | logical tokens/s: 75681.7 | valid tokens/s: 75681.7 | physical tokens/s: 76821.9 | loss tokens/s: 58085.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.269855E-01 | lm-global-supervised-tokens: 1.287636E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.278296E-02 | lm-token-buffer-mib: 2.360391E+02 | global_load_balancing_loss: 9.996835E-01 | loss scale: 1.0 | grad norm: 0.170 | params norm: 11200.704 | params rms: 0.060648 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:58:33.039560 | finish at 2026-10-17 17:31:55 + [2026-10-13 07:37:03] iteration 664/ 2384 | consumed samples: 21248 | elapsed time per iteration (ms): 220832.8 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 75972.3 | valid tokens/s: 75972.3 | physical tokens/s: 76453.0 | loss tokens/s: 58887.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.348532E-01 | lm-global-supervised-tokens: 1.300424E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.630024E-02 | lm-token-buffer-mib: 2.337663E+02 | global_load_balancing_loss: 9.986721E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11200.670 | params rms: 0.060648 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:30:32.409286 | finish at 2026-10-17 17:07:35 + [2026-10-13 07:40:43] iteration 665/ 2384 | consumed samples: 21280 | elapsed time per iteration (ms): 219738.6 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 76350.6 | valid tokens/s: 76350.6 | physical tokens/s: 76851.5 | loss tokens/s: 57347.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.592218E-01 | lm-global-supervised-tokens: 1.260147E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.673127E-02 | lm-token-buffer-mib: 2.340905E+02 | global_load_balancing_loss: 1.000639E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11200.634 | params rms: 0.060648 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:55:30.624328 | finish at 2026-10-17 16:36:13 + [2026-10-13 07:44:24] iteration 666/ 2384 | consumed samples: 21312 | elapsed time per iteration (ms): 221495.4 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 75745.0 | valid tokens/s: 75745.0 | physical tokens/s: 76022.3 | loss tokens/s: 55367.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.874415E-01 | lm-global-supervised-tokens: 1.226365E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.201132E-02 | lm-token-buffer-mib: 2.328575E+02 | global_load_balancing_loss: 1.001720E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11200.592 | params rms: 0.060648 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:42:09.115959 | finish at 2026-10-17 17:26:33 + [2026-10-13 07:48:09] iteration 667/ 2384 | consumed samples: 21344 | elapsed time per iteration (ms): 224558.7 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 74711.8 | valid tokens/s: 74711.8 | physical tokens/s: 75348.3 | loss tokens/s: 54443.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.882483E-01 | lm-global-supervised-tokens: 1.222579E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.471407E-02 | lm-token-buffer-mib: 2.332906E+02 | global_load_balancing_loss: 1.004149E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11200.555 | params rms: 0.060647 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:06:07.370328 | finish at 2026-10-17 18:54:16 + [2026-10-13 07:51:52] iteration 668/ 2384 | consumed samples: 21376 | elapsed time per iteration (ms): 223122.2 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 75192.9 | valid tokens/s: 75192.9 | physical tokens/s: 76005.6 | loss tokens/s: 56264.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.135694E-01 | lm-global-supervised-tokens: 1.255377E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.573109E-02 | lm-token-buffer-mib: 2.350883E+02 | global_load_balancing_loss: 1.000986E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11200.522 | params rms: 0.060647 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:21:17.707904 | finish at 2026-10-17 18:13:10 + [2026-10-13 07:55:34] iteration 669/ 2384 | consumed samples: 21408 | elapsed time per iteration (ms): 222040.4 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 75559.2 | valid tokens/s: 75559.2 | physical tokens/s: 75829.1 | loss tokens/s: 59681.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.825950E-01 | lm-global-supervised-tokens: 1.325170E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.505486E-02 | lm-token-buffer-mib: 2.330491E+02 | global_load_balancing_loss: 1.000938E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11200.491 | params rms: 0.060647 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:46:39.238617 | finish at 2026-10-17 17:42:13 + [2026-10-13 07:59:43] iteration 670/ 2384 | consumed samples: 21440 | elapsed time per iteration (ms): 249331.7 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.24% | FLOPs estimator: workload-aware | logical tokens/s: 67288.7 | valid tokens/s: 67288.7 | physical tokens/s: 68470.3 | loss tokens/s: 54968.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.389323E-01 | lm-global-supervised-tokens: 1.370534E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.298723E-02 | lm-token-buffer-mib: 2.386116E+02 | global_load_balancing_loss: 1.001328E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11200.458 | params rms: 0.060647 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:42:34.464478 | finish at 2026-10-18 06:42:18 + [2026-10-13 08:03:24] iteration 671/ 2384 | consumed samples: 21472 | elapsed time per iteration (ms): 220815.5 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 75978.3 | valid tokens/s: 75978.3 | physical tokens/s: 76597.0 | loss tokens/s: 52889.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.124175E-01 | lm-global-supervised-tokens: 1.167874E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.522256E-02 | lm-token-buffer-mib: 2.337910E+02 | global_load_balancing_loss: 9.996966E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11200.427 | params rms: 0.060647 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:04:16.868220 | finish at 2026-10-17 17:07:41 + [2026-10-13 08:07:02] iteration 672/ 2384 | consumed samples: 21504 | elapsed time per iteration (ms): 218123.8 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 76916.0 | valid tokens/s: 76916.0 | physical tokens/s: 77402.2 | loss tokens/s: 57584.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.818440E-01 | lm-global-supervised-tokens: 1.256058E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.914552E-02 | lm-token-buffer-mib: 2.339250E+02 | global_load_balancing_loss: 1.000891E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11200.393 | params rms: 0.060646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:43:47.864849 | finish at 2026-10-17 15:50:50 + [2026-10-13 08:10:45] iteration 673/ 2384 | consumed samples: 21536 | elapsed time per iteration (ms): 222403.8 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.64% | FLOPs estimator: workload-aware | logical tokens/s: 75435.7 | valid tokens/s: 75435.7 | physical tokens/s: 75925.0 | loss tokens/s: 56169.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.021948E-01 | lm-global-supervised-tokens: 1.249229E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.340824E-02 | lm-token-buffer-mib: 2.328307E+02 | global_load_balancing_loss: 9.997603E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11200.358 | params rms: 0.060646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:42:12.875710 | finish at 2026-10-17 17:52:57 + [2026-10-13 08:14:29] iteration 674/ 2384 | consumed samples: 21568 | elapsed time per iteration (ms): 223958.5 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 74912.1 | valid tokens/s: 74912.1 | physical tokens/s: 75763.5 | loss tokens/s: 58575.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.927401E-01 | lm-global-supervised-tokens: 1.311836E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.192082E-02 | lm-token-buffer-mib: 2.344890E+02 | global_load_balancing_loss: 9.990363E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11200.322 | params rms: 0.060646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:22:49.057674 | finish at 2026-10-17 18:37:18 + [2026-10-13 08:18:12] iteration 675/ 2384 | consumed samples: 21600 | elapsed time per iteration (ms): 223730.9 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 74988.2 | valid tokens/s: 74988.2 | physical tokens/s: 75717.0 | loss tokens/s: 55955.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.140324E-01 | lm-global-supervised-tokens: 1.251889E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.257861E-02 | lm-token-buffer-mib: 2.342758E+02 | global_load_balancing_loss: 9.996659E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11200.286 | params rms: 0.060646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:12:36.151375 | finish at 2026-10-17 18:30:48 + [2026-10-13 08:21:56] iteration 676/ 2384 | consumed samples: 21632 | elapsed time per iteration (ms): 224123.0 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 74857.0 | valid tokens/s: 74857.0 | physical tokens/s: 75697.0 | loss tokens/s: 55413.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.839564E-01 | lm-global-supervised-tokens: 1.241936E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.632275E-02 | lm-token-buffer-mib: 2.366333E+02 | global_load_balancing_loss: 1.001561E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11200.250 | params rms: 0.060646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:20:02.042711 | finish at 2026-10-17 18:41:58 + [2026-10-13 08:25:38] iteration 677/ 2384 | consumed samples: 21664 | elapsed time per iteration (ms): 222076.5 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 75547.0 | valid tokens/s: 75547.0 | physical tokens/s: 76306.4 | loss tokens/s: 55355.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.847744E-01 | lm-global-supervised-tokens: 1.229325E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.419573E-02 | lm-token-buffer-mib: 2.339264E+02 | global_load_balancing_loss: 1.002855E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11200.215 | params rms: 0.060646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:18:04.591501 | finish at 2026-10-17 17:43:43 + [2026-10-13 08:29:20] iteration 678/ 2384 | consumed samples: 21696 | elapsed time per iteration (ms): 221033.0 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 75903.5 | valid tokens/s: 75903.5 | physical tokens/s: 76193.2 | loss tokens/s: 52750.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.840025E-01 | lm-global-supervised-tokens: 1.165961E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.460272E-02 | lm-token-buffer-mib: 2.329827E+02 | global_load_balancing_loss: 1.001320E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11200.181 | params rms: 0.060645 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:44:42.306122 | finish at 2026-10-17 17:14:02 + [2026-10-13 08:33:03] iteration 679/ 2384 | consumed samples: 21728 | elapsed time per iteration (ms): 223027.3 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.36% | FLOPs estimator: workload-aware | logical tokens/s: 75224.8 | valid tokens/s: 75224.8 | physical tokens/s: 75959.9 | loss tokens/s: 60329.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.209194E-01 | lm-global-supervised-tokens: 1.345521E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.343098E-02 | lm-token-buffer-mib: 2.343170E+02 | global_load_balancing_loss: 1.001697E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11200.144 | params rms: 0.060645 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:37:41.502801 | finish at 2026-10-17 18:10:44 + [2026-10-13 08:37:06] iteration 680/ 2384 | consumed samples: 21760 | elapsed time per iteration (ms): 243597.4 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 68872.6 | valid tokens/s: 68872.6 | physical tokens/s: 69764.7 | loss tokens/s: 49688.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.316605E-01 | lm-global-supervised-tokens: 1.210393E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.382981E-02 | lm-token-buffer-mib: 2.344369E+02 | global_load_balancing_loss: 9.995846E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11200.109 | params rms: 0.060645 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:18:09.993484 | finish at 2026-10-18 03:55:16 +times across ranks (ms): + save-checkpoint: + rank 0: 24380.58 + rank 1: 24380.54 + rank 2: 24380.57 + rank 3: 24380.52 + rank 4: 24380.52 + rank 5: 24380.54 + rank 6: 24380.55 + rank 7: 24380.54 + rank 8: 24380.42 + rank 9: 24380.44 + rank 10: 24380.46 + rank 11: 24380.45 + rank 12: 24380.43 + rank 13: 24380.42 + rank 14: 24380.42 + rank 15: 24380.41 + rank 16: 24380.49 + rank 17: 24380.54 + rank 18: 24380.52 + rank 19: 24380.51 + rank 20: 24380.45 + rank 21: 24380.49 + rank 22: 24380.49 + rank 23: 24380.52 + rank 24: 24380.73 + rank 25: 24380.76 + rank 26: 24380.71 + rank 27: 24381.29 + rank 28: 24380.75 + rank 29: 24380.73 + rank 30: 24380.73 + rank 31: 24380.73 + rank 32: 24380.54 + rank 33: 24380.54 + rank 34: 24380.57 + rank 35: 24380.68 + rank 36: 24380.63 + rank 37: 24380.73 + rank 38: 24380.46 + rank 39: 24380.61 + rank 40: 24380.11 + rank 41: 24380.30 + rank 42: 24380.27 + rank 43: 24380.51 + rank 44: 24380.31 + rank 45: 24380.27 + rank 46: 24380.27 + rank 47: 24380.25 + rank 48: 24380.44 + rank 49: 24380.45 + rank 50: 24380.46 + rank 51: 24380.40 + rank 52: 24380.44 + rank 53: 24380.39 + rank 54: 24380.48 + rank 55: 24380.49 + rank 56: 24380.67 + rank 57: 24380.68 + rank 58: 24380.68 + rank 59: 24380.67 + rank 60: 24380.53 + rank 61: 24380.71 + rank 62: 24380.65 + rank 63: 24380.64 + [2026-10-13 08:41:14] iteration 681/ 2384 | consumed samples: 21792 | elapsed time per iteration (ms): 223222.1 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75159.1 | valid tokens/s: 75159.1 | physical tokens/s: 76648.5 | loss tokens/s: 61488.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.253965E-01 | lm-global-supervised-tokens: 1.372563E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.132287E-02 | lm-token-buffer-mib: 2.373254E+02 | global_load_balancing_loss: 9.996272E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11200.073 | params rms: 0.060645 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:35:47.211159 | finish at 2026-10-17 18:17:01 + [2026-10-13 08:44:58] iteration 682/ 2384 | consumed samples: 21824 | elapsed time per iteration (ms): 224540.1 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 74718.1 | valid tokens/s: 74718.1 | physical tokens/s: 75197.0 | loss tokens/s: 54539.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.197526E-01 | lm-global-supervised-tokens: 1.224633E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.803936E-02 | lm-token-buffer-mib: 2.333471E+02 | global_load_balancing_loss: 9.981944E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11200.037 | params rms: 0.060645 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:09:27.269032 | finish at 2026-10-17 18:54:26 + [2026-10-13 08:48:39] iteration 683/ 2384 | consumed samples: 21856 | elapsed time per iteration (ms): 220920.4 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 75942.2 | valid tokens/s: 75942.2 | physical tokens/s: 76400.9 | loss tokens/s: 56879.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.756373E-01 | lm-global-supervised-tokens: 1.256573E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.533832E-02 | lm-token-buffer-mib: 2.346188E+02 | global_load_balancing_loss: 9.981328E-01 | loss scale: 1.0 | grad norm: 0.205 | params norm: 11200.003 | params rms: 0.060644 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:23:05.609159 | finish at 2026-10-17 17:11:45 + [2026-10-13 08:52:24] iteration 684/ 2384 | consumed samples: 21888 | elapsed time per iteration (ms): 224623.4 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 74690.0 | valid tokens/s: 74690.0 | physical tokens/s: 75213.5 | loss tokens/s: 58970.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.478015E-01 | lm-global-supervised-tokens: 1.324607E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.712833E-02 | lm-token-buffer-mib: 2.339459E+02 | global_load_balancing_loss: 1.001039E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11199.964 | params rms: 0.060644 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:04:19.760094 | finish at 2026-10-17 18:56:44 + [2026-10-13 08:56:07] iteration 685/ 2384 | consumed samples: 21920 | elapsed time per iteration (ms): 223273.5 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75141.9 | valid tokens/s: 75141.9 | physical tokens/s: 76019.4 | loss tokens/s: 58590.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.737911E-01 | lm-global-supervised-tokens: 1.308173E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.356131E-02 | lm-token-buffer-mib: 2.348139E+02 | global_load_balancing_loss: 1.000999E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11199.927 | params rms: 0.060644 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:22:21.712898 | finish at 2026-10-17 18:18:29 + [2026-10-13 08:59:51] iteration 686/ 2384 | consumed samples: 21952 | elapsed time per iteration (ms): 224023.8 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 74890.3 | valid tokens/s: 74890.3 | physical tokens/s: 75549.4 | loss tokens/s: 56831.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.555049E-01 | lm-global-supervised-tokens: 1.273170E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.402681E-02 | lm-token-buffer-mib: 2.333325E+02 | global_load_balancing_loss: 1.001444E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11199.894 | params rms: 0.060644 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:39:52.415058 | finish at 2026-10-17 18:39:44 + [2026-10-13 09:03:34] iteration 687/ 2384 | consumed samples: 21984 | elapsed time per iteration (ms): 222966.1 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 75245.5 | valid tokens/s: 75245.5 | physical tokens/s: 76199.7 | loss tokens/s: 60584.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.132086E-01 | lm-global-supervised-tokens: 1.350822E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.748041E-02 | lm-token-buffer-mib: 2.363715E+02 | global_load_balancing_loss: 1.002371E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11199.863 | params rms: 0.060644 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:06:13.553390 | finish at 2026-10-17 18:09:48 + [2026-10-13 09:07:18] iteration 688/ 2384 | consumed samples: 22016 | elapsed time per iteration (ms): 223793.6 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 74967.2 | valid tokens/s: 74967.2 | physical tokens/s: 75658.3 | loss tokens/s: 58925.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.126098E-01 | lm-global-supervised-tokens: 1.318711E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.263505E-02 | lm-token-buffer-mib: 2.354811E+02 | global_load_balancing_loss: 1.002985E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11199.832 | params rms: 0.060643 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:25:53.937653 | finish at 2026-10-17 18:33:12 + [2026-10-13 09:11:01] iteration 689/ 2384 | consumed samples: 22048 | elapsed time per iteration (ms): 223007.9 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 75231.4 | valid tokens/s: 75231.4 | physical tokens/s: 75718.8 | loss tokens/s: 56362.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.965322E-01 | lm-global-supervised-tokens: 1.256933E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.285231E-02 | lm-token-buffer-mib: 2.335854E+02 | global_load_balancing_loss: 1.000750E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11199.799 | params rms: 0.060643 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:59:58.335943 | finish at 2026-10-17 18:10:59 + [2026-10-13 09:15:07] iteration 690/ 2384 | consumed samples: 22080 | elapsed time per iteration (ms): 245849.5 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 68241.8 | valid tokens/s: 68241.8 | physical tokens/s: 68834.3 | loss tokens/s: 50552.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.833364E-01 | lm-global-supervised-tokens: 1.242838E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.673983E-02 | lm-token-buffer-mib: 2.326450E+02 | global_load_balancing_loss: 1.000684E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11199.765 | params rms: 0.060643 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:41:09.067036 | finish at 2026-10-18 04:56:16 + [2026-10-13 09:18:51] iteration 691/ 2384 | consumed samples: 22112 | elapsed time per iteration (ms): 224372.2 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 74774.0 | valid tokens/s: 74773.9 | physical tokens/s: 75753.6 | loss tokens/s: 59149.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.293000E-01 | lm-global-supervised-tokens: 1.327145E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.740435E-02 | lm-token-buffer-mib: 2.354397E+02 | global_load_balancing_loss: 9.978706E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11199.731 | params rms: 0.060643 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:31:02.106366 | finish at 2026-10-17 18:49:53 + [2026-10-13 09:22:34] iteration 692/ 2384 | consumed samples: 22144 | elapsed time per iteration (ms): 223084.1 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75205.7 | valid tokens/s: 75205.7 | physical tokens/s: 75553.2 | loss tokens/s: 50953.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.404985E-01 | lm-global-supervised-tokens: 1.136688E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.780666E-02 | lm-token-buffer-mib: 2.330953E+02 | global_load_balancing_loss: 9.981178E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11199.697 | params rms: 0.060643 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:50:58.333923 | finish at 2026-10-17 18:13:33 + [2026-10-13 09:26:19] iteration 693/ 2384 | consumed samples: 22176 | elapsed time per iteration (ms): 225146.6 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 74516.8 | valid tokens/s: 74516.8 | physical tokens/s: 75983.9 | loss tokens/s: 59619.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.782528E-01 | lm-global-supervised-tokens: 1.342323E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.538941E-02 | lm-token-buffer-mib: 2.384399E+02 | global_load_balancing_loss: 9.979254E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11199.661 | params rms: 0.060643 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:45:22.928432 | finish at 2026-10-17 19:11:42 + [2026-10-13 09:30:03] iteration 694/ 2384 | consumed samples: 22208 | elapsed time per iteration (ms): 223155.2 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.04% | FLOPs estimator: workload-aware | logical tokens/s: 75181.7 | valid tokens/s: 75181.6 | physical tokens/s: 75760.9 | loss tokens/s: 59594.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.714948E-01 | lm-global-supervised-tokens: 1.329874E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.534382E-02 | lm-token-buffer-mib: 2.352128E+02 | global_load_balancing_loss: 1.001009E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11199.627 | params rms: 0.060642 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:45:32.248521 | finish at 2026-10-17 18:15:35 + [2026-10-13 09:33:45] iteration 695/ 2384 | consumed samples: 22240 | elapsed time per iteration (ms): 221990.9 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 75576.0 | valid tokens/s: 75576.0 | physical tokens/s: 76262.9 | loss tokens/s: 50839.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.871136E-01 | lm-global-supervised-tokens: 1.128600E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.142147E-02 | lm-token-buffer-mib: 2.324062E+02 | global_load_balancing_loss: 1.002451E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11199.596 | params rms: 0.060642 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:09:02.572985 | finish at 2026-10-17 17:42:47 + [2026-10-13 09:37:27] iteration 696/ 2384 | consumed samples: 22272 | elapsed time per iteration (ms): 222231.7 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75494.2 | valid tokens/s: 75494.2 | physical tokens/s: 76124.4 | loss tokens/s: 48184.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.616222E-01 | lm-global-supervised-tokens: 1.070815E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.390242E-02 | lm-token-buffer-mib: 2.337416E+02 | global_load_balancing_loss: 1.000780E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11199.565 | params rms: 0.060642 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:12:07.130835 | finish at 2026-10-17 17:49:34 + [2026-10-13 09:41:12] iteration 697/ 2384 | consumed samples: 22304 | elapsed time per iteration (ms): 225470.2 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.02% | FLOPs estimator: workload-aware | logical tokens/s: 74409.7 | valid tokens/s: 74409.7 | physical tokens/s: 75252.7 | loss tokens/s: 50325.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.665372E-01 | lm-global-supervised-tokens: 1.134688E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.733702E-02 | lm-token-buffer-mib: 2.332947E+02 | global_load_balancing_loss: 1.000680E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11199.530 | params rms: 0.060642 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:39:28.248017 | finish at 2026-10-17 19:20:40 + [2026-10-13 09:44:58] iteration 698/ 2384 | consumed samples: 22336 | elapsed time per iteration (ms): 225520.8 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 74393.2 | valid tokens/s: 74393.2 | physical tokens/s: 75098.7 | loss tokens/s: 54954.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.425840E-01 | lm-global-supervised-tokens: 1.239343E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.572190E-02 | lm-token-buffer-mib: 2.350925E+02 | global_load_balancing_loss: 1.000938E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11199.489 | params rms: 0.060642 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:37:08.106777 | finish at 2026-10-17 19:22:06 + [2026-10-13 09:48:39] iteration 699/ 2384 | consumed samples: 22368 | elapsed time per iteration (ms): 220955.0 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 75930.5 | valid tokens/s: 75930.5 | physical tokens/s: 76427.5 | loss tokens/s: 58087.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.573003E-01 | lm-global-supervised-tokens: 1.283470E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.400715E-02 | lm-token-buffer-mib: 2.323359E+02 | global_load_balancing_loss: 9.992786E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11199.448 | params rms: 0.060641 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:25:09.182504 | finish at 2026-10-17 17:13:48 + [2026-10-13 09:52:42] iteration 700/ 2384 | consumed samples: 22400 | elapsed time per iteration (ms): 242818.6 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 69093.5 | valid tokens/s: 69093.5 | physical tokens/s: 69703.5 | loss tokens/s: 52097.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.714126E-01 | lm-global-supervised-tokens: 1.265017E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.683836E-02 | lm-token-buffer-mib: 2.334579E+02 | global_load_balancing_loss: 9.995534E-01 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11199.407 | params rms: 0.060641 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:35:06.584530 | finish at 2026-10-18 03:27:48 + [2026-10-13 09:56:25] iteration 701/ 2384 | consumed samples: 22432 | elapsed time per iteration (ms): 223007.3 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 75231.6 | valid tokens/s: 75231.6 | physical tokens/s: 76074.0 | loss tokens/s: 55617.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.754493E-01 | lm-global-supervised-tokens: 1.240318E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.663826E-02 | lm-token-buffer-mib: 2.352243E+02 | global_load_balancing_loss: 1.001851E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11199.365 | params rms: 0.060641 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:15:21.288942 | finish at 2026-10-17 18:11:46 + [2026-10-13 10:00:07] iteration 702/ 2384 | consumed samples: 22464 | elapsed time per iteration (ms): 222740.4 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75321.8 | valid tokens/s: 75321.8 | physical tokens/s: 76459.4 | loss tokens/s: 56317.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.861268E-01 | lm-global-supervised-tokens: 1.254417E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.769516E-02 | lm-token-buffer-mib: 2.339440E+02 | global_load_balancing_loss: 1.001066E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11199.317 | params rms: 0.060641 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:04:09.310019 | finish at 2026-10-17 18:04:17 + [2026-10-13 10:03:51] iteration 703/ 2384 | consumed samples: 22496 | elapsed time per iteration (ms): 223917.6 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 74925.8 | valid tokens/s: 74925.8 | physical tokens/s: 75766.7 | loss tokens/s: 59217.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.878654E-01 | lm-global-supervised-tokens: 1.325976E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.736522E-02 | lm-token-buffer-mib: 2.347597E+02 | global_load_balancing_loss: 1.002565E+00 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11199.267 | params rms: 0.060640 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:33:25.456203 | finish at 2026-10-17 18:37:17 + [2026-10-13 10:07:35] iteration 704/ 2384 | consumed samples: 22528 | elapsed time per iteration (ms): 224074.9 | throughput per GPU (TFLOP/s/GPU): 25.3 | MFU 8.12% | FLOPs estimator: workload-aware | logical tokens/s: 74873.0 | valid tokens/s: 74873.0 | physical tokens/s: 75313.1 | loss tokens/s: 61635.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.192227E-01 | lm-global-supervised-tokens: 1.381101E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.236128E-02 | lm-token-buffer-mib: 2.327318E+02 | global_load_balancing_loss: 1.001210E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11199.219 | params rms: 0.060640 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:34:05.789394 | finish at 2026-10-17 18:41:41 + [2026-10-13 10:11:18] iteration 705/ 2384 | consumed samples: 22560 | elapsed time per iteration (ms): 222577.6 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75376.9 | valid tokens/s: 75376.9 | physical tokens/s: 75982.8 | loss tokens/s: 59490.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.095379E-01 | lm-global-supervised-tokens: 1.324124E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.618122E-02 | lm-token-buffer-mib: 2.340089E+02 | global_load_balancing_loss: 1.000076E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11199.171 | params rms: 0.060640 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:48:27.719950 | finish at 2026-10-17 17:59:46 + [2026-10-13 10:15:03] iteration 706/ 2384 | consumed samples: 22592 | elapsed time per iteration (ms): 225208.2 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.81% | FLOPs estimator: workload-aware | logical tokens/s: 74496.4 | valid tokens/s: 74496.4 | physical tokens/s: 75021.9 | loss tokens/s: 53590.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.328448E-01 | lm-global-supervised-tokens: 1.206892E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.420769E-02 | lm-token-buffer-mib: 2.343698E+02 | global_load_balancing_loss: 9.988647E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11199.123 | params rms: 0.060640 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:58:19.287151 | finish at 2026-10-17 19:13:22 + [2026-10-13 10:18:47] iteration 707/ 2384 | consumed samples: 22624 | elapsed time per iteration (ms): 224010.4 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 74894.5 | valid tokens/s: 74894.5 | physical tokens/s: 75343.9 | loss tokens/s: 58844.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.456110E-01 | lm-global-supervised-tokens: 1.318182E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.333982E-02 | lm-token-buffer-mib: 2.332017E+02 | global_load_balancing_loss: 9.986701E-01 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11199.075 | params rms: 0.060639 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:21:05.490074 | finish at 2026-10-17 18:39:53 + [2026-10-13 10:22:32] iteration 708/ 2384 | consumed samples: 22656 | elapsed time per iteration (ms): 224841.9 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 74617.7 | valid tokens/s: 74617.7 | physical tokens/s: 75358.5 | loss tokens/s: 57220.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 9.188565E-01 | lm-global-supervised-tokens: 1.286549E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.512305E-02 | lm-token-buffer-mib: 2.359583E+02 | global_load_balancing_loss: 9.987684E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11199.027 | params rms: 0.060639 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:40:34.989421 | finish at 2026-10-17 19:03:07 + [2026-10-13 10:26:14] iteration 709/ 2384 | consumed samples: 22688 | elapsed time per iteration (ms): 222053.9 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 75554.7 | valid tokens/s: 75554.7 | physical tokens/s: 76592.3 | loss tokens/s: 58275.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.716498E-01 | lm-global-supervised-tokens: 1.294030E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.318113E-02 | lm-token-buffer-mib: 2.363135E+02 | global_load_balancing_loss: 1.000369E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11198.978 | params rms: 0.060639 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:19:00.202236 | finish at 2026-10-17 17:45:14 + [2026-10-13 10:30:19] iteration 710/ 2384 | consumed samples: 22720 | elapsed time per iteration (ms): 244722.1 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 68556.1 | valid tokens/s: 68556.1 | physical tokens/s: 69457.4 | loss tokens/s: 53042.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.472138E-01 | lm-global-supervised-tokens: 1.298060E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.769137E-02 | lm-token-buffer-mib: 2.353078E+02 | global_load_balancing_loss: 1.001025E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11198.930 | params rms: 0.060639 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:47:44.782661 | finish at 2026-10-18 04:18:03 + [2026-10-13 10:34:02] iteration 711/ 2384 | consumed samples: 22752 | elapsed time per iteration (ms): 223310.3 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75129.6 | valid tokens/s: 75129.6 | physical tokens/s: 75653.2 | loss tokens/s: 54978.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.389422E-01 | lm-global-supervised-tokens: 1.227731E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.660290E-02 | lm-token-buffer-mib: 2.338969E+02 | global_load_balancing_loss: 1.002148E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11198.881 | params rms: 0.060638 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:46:38.106160 | finish at 2026-10-17 18:20:40 + [2026-10-13 10:37:46] iteration 712/ 2384 | consumed samples: 22784 | elapsed time per iteration (ms): 223670.3 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75008.6 | valid tokens/s: 75008.6 | physical tokens/s: 75876.0 | loss tokens/s: 59111.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.731321E-01 | lm-global-supervised-tokens: 1.322144E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.480468E-02 | lm-token-buffer-mib: 2.346193E+02 | global_load_balancing_loss: 1.001066E+00 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11198.833 | params rms: 0.060638 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:52:56.754368 | finish at 2026-10-17 18:30:42 + [2026-10-13 10:41:27] iteration 713/ 2384 | consumed samples: 22816 | elapsed time per iteration (ms): 221069.9 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 75890.9 | valid tokens/s: 75890.9 | physical tokens/s: 76501.5 | loss tokens/s: 60874.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.971945E-01 | lm-global-supervised-tokens: 1.345760E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.700819E-02 | lm-token-buffer-mib: 2.333793E+02 | global_load_balancing_loss: 9.997233E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11198.783 | params rms: 0.060638 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:36:47.874671 | finish at 2026-10-17 17:18:15 + [2026-10-13 10:45:08] iteration 714/ 2384 | consumed samples: 22848 | elapsed time per iteration (ms): 220862.6 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 6.01% | FLOPs estimator: workload-aware | logical tokens/s: 75962.1 | valid tokens/s: 75962.1 | physical tokens/s: 76902.3 | loss tokens/s: 60354.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.294100E-01 | lm-global-supervised-tokens: 1.332994E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.416495E-02 | lm-token-buffer-mib: 2.349800E+02 | global_load_balancing_loss: 1.000277E+00 | loss scale: 1.0 | grad norm: 0.208 | params norm: 11198.731 | params rms: 0.060637 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:27:20.511091 | finish at 2026-10-17 17:12:28 + [2026-10-13 10:48:50] iteration 715/ 2384 | consumed samples: 22880 | elapsed time per iteration (ms): 222852.8 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 75283.8 | valid tokens/s: 75283.8 | physical tokens/s: 76208.3 | loss tokens/s: 60103.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.101022E-01 | lm-global-supervised-tokens: 1.339412E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.431685E-02 | lm-token-buffer-mib: 2.340569E+02 | global_load_balancing_loss: 9.998237E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11198.680 | params rms: 0.060637 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:19:01.344110 | finish at 2026-10-17 18:07:52 + [2026-10-13 10:52:34] iteration 716/ 2384 | consumed samples: 22912 | elapsed time per iteration (ms): 223594.7 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 75034.0 | valid tokens/s: 75034.0 | physical tokens/s: 76134.6 | loss tokens/s: 54991.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.232859E-01 | lm-global-supervised-tokens: 1.229583E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.634140E-02 | lm-token-buffer-mib: 2.371725E+02 | global_load_balancing_loss: 1.000507E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11198.627 | params rms: 0.060637 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:35:55.944652 | finish at 2026-10-17 18:28:30 + [2026-10-13 10:56:17] iteration 717/ 2384 | consumed samples: 22944 | elapsed time per iteration (ms): 223120.7 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 75193.2 | valid tokens/s: 75193.2 | physical tokens/s: 76054.6 | loss tokens/s: 61481.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.793550E-01 | lm-global-supervised-tokens: 1.371771E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.292925E-02 | lm-token-buffer-mib: 2.348579E+02 | global_load_balancing_loss: 1.000850E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11198.577 | params rms: 0.060637 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:19:02.153049 | finish at 2026-10-17 18:15:19 + [2026-10-13 11:00:01] iteration 718/ 2384 | consumed samples: 22976 | elapsed time per iteration (ms): 224120.1 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 74858.0 | valid tokens/s: 74858.0 | physical tokens/s: 75286.2 | loss tokens/s: 51802.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.849596E-01 | lm-global-supervised-tokens: 1.160992E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.526359E-02 | lm-token-buffer-mib: 2.336387E+02 | global_load_balancing_loss: 1.001067E+00 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11198.527 | params rms: 0.060636 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:43:04.083061 | finish at 2026-10-17 18:43:05 + [2026-10-13 11:03:46] iteration 719/ 2384 | consumed samples: 23008 | elapsed time per iteration (ms): 224795.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 74633.2 | valid tokens/s: 74633.2 | physical tokens/s: 75603.2 | loss tokens/s: 58189.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.666698E-01 | lm-global-supervised-tokens: 1.308076E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.690933E-02 | lm-token-buffer-mib: 2.352645E+02 | global_load_balancing_loss: 1.000143E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11198.478 | params rms: 0.060636 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:58:04.469855 | finish at 2026-10-17 19:01:51 + [2026-10-13 11:07:55] iteration 720/ 2384 | consumed samples: 23040 | elapsed time per iteration (ms): 248995.4 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 67379.6 | valid tokens/s: 67379.6 | physical tokens/s: 68130.2 | loss tokens/s: 52031.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.237492E-01 | lm-global-supervised-tokens: 1.295557E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.356264E-02 | lm-token-buffer-mib: 2.345708E+02 | global_load_balancing_loss: 1.003127E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11198.430 | params rms: 0.060636 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:05:28.349487 | finish at 2026-10-18 06:13:23 +times across ranks (ms): + save-checkpoint: + rank 0: 24106.88 + rank 1: 24106.91 + rank 2: 24106.89 + rank 3: 24106.92 + rank 4: 24106.89 + rank 5: 24106.91 + rank 6: 24106.91 + rank 7: 24106.88 + rank 8: 24106.98 + rank 9: 24107.03 + rank 10: 24106.97 + rank 11: 24106.95 + rank 12: 24106.94 + rank 13: 24106.94 + rank 14: 24106.96 + rank 15: 24106.94 + rank 16: 24107.07 + rank 17: 24107.03 + rank 18: 24107.05 + rank 19: 24107.02 + rank 20: 24107.04 + rank 21: 24107.06 + rank 22: 24107.03 + rank 23: 24107.04 + rank 24: 24107.53 + rank 25: 24107.32 + rank 26: 24107.27 + rank 27: 24107.33 + rank 28: 24107.36 + rank 29: 24107.33 + rank 30: 24107.36 + rank 31: 24107.33 + rank 32: 24106.70 + rank 33: 24106.74 + rank 34: 24106.68 + rank 35: 24106.73 + rank 36: 24106.67 + rank 37: 24106.68 + rank 38: 24106.66 + rank 39: 24106.67 + rank 40: 24107.16 + rank 41: 24107.15 + rank 42: 24107.15 + rank 43: 24107.17 + rank 44: 24107.16 + rank 45: 24107.15 + rank 46: 24107.15 + rank 47: 24107.15 + rank 48: 24106.26 + rank 49: 24106.27 + rank 50: 24106.27 + rank 51: 24106.24 + rank 52: 24106.25 + rank 53: 24106.27 + rank 54: 24106.24 + rank 55: 24106.24 + rank 56: 24107.18 + rank 57: 24107.20 + rank 58: 24107.19 + rank 59: 24107.16 + rank 60: 24107.18 + rank 61: 24107.20 + rank 62: 24107.16 + rank 63: 24107.19 + [2026-10-13 11:11:59] iteration 721/ 2384 | consumed samples: 23072 | elapsed time per iteration (ms): 219434.9 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 76456.4 | valid tokens/s: 76456.3 | physical tokens/s: 77223.5 | loss tokens/s: 60062.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.131179E-01 | lm-global-supervised-tokens: 1.317983E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.700390E-01 | lm-token-buffer-mib: 2.336890E+02 | global_load_balancing_loss: 9.983805E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11198.385 | params rms: 0.060636 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:22:00.272081 | finish at 2026-10-17 16:33:59 + [2026-10-13 11:15:44] iteration 722/ 2384 | consumed samples: 23104 | elapsed time per iteration (ms): 225004.2 | throughput per GPU (TFLOP/s/GPU): 25.5 | MFU 8.16% | FLOPs estimator: workload-aware | logical tokens/s: 74563.8 | valid tokens/s: 74563.8 | physical tokens/s: 74882.0 | loss tokens/s: 54743.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.054846E-01 | lm-global-supervised-tokens: 1.231754E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.523003E-02 | lm-token-buffer-mib: 2.326988E+02 | global_load_balancing_loss: 1.000166E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11198.335 | params rms: 0.060635 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:52:36.906271 | finish at 2026-10-17 19:08:21 + [2026-10-13 11:19:28] iteration 723/ 2384 | consumed samples: 23136 | elapsed time per iteration (ms): 224145.6 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 74849.5 | valid tokens/s: 74849.5 | physical tokens/s: 75745.6 | loss tokens/s: 61029.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.113627E-01 | lm-global-supervised-tokens: 1.367944E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.281968E-02 | lm-token-buffer-mib: 2.353433E+02 | global_load_balancing_loss: 9.989891E-01 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11198.283 | params rms: 0.060635 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:25:05.867871 | finish at 2026-10-17 18:44:34 + [2026-10-13 11:23:10] iteration 724/ 2384 | consumed samples: 23168 | elapsed time per iteration (ms): 222107.7 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 75536.2 | valid tokens/s: 75536.2 | physical tokens/s: 75838.4 | loss tokens/s: 55195.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.442790E-01 | lm-global-supervised-tokens: 1.225943E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.558326E-02 | lm-token-buffer-mib: 2.331771E+02 | global_load_balancing_loss: 9.996031E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11198.233 | params rms: 0.060635 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:24:58.818197 | finish at 2026-10-17 17:48:09 + [2026-10-13 11:26:54] iteration 725/ 2384 | consumed samples: 23200 | elapsed time per iteration (ms): 223742.4 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 74984.5 | valid tokens/s: 74984.5 | physical tokens/s: 75899.3 | loss tokens/s: 57287.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.869190E-01 | lm-global-supervised-tokens: 1.281773E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.198543E-02 | lm-token-buffer-mib: 2.363057E+02 | global_load_balancing_loss: 9.994869E-01 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11198.185 | params rms: 0.060635 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:06:28.568311 | finish at 2026-10-17 18:33:22 + [2026-10-13 11:30:37] iteration 726/ 2384 | consumed samples: 23232 | elapsed time per iteration (ms): 223016.4 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 75228.4 | valid tokens/s: 75228.4 | physical tokens/s: 76054.0 | loss tokens/s: 57891.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.906678E-01 | lm-global-supervised-tokens: 1.291074E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.185828E-02 | lm-token-buffer-mib: 2.362969E+02 | global_load_balancing_loss: 1.001128E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11198.136 | params rms: 0.060634 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:42:41.167646 | finish at 2026-10-17 18:13:18 + [2026-10-13 11:34:23] iteration 727/ 2384 | consumed samples: 23264 | elapsed time per iteration (ms): 226271.4 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 74146.2 | valid tokens/s: 74146.2 | physical tokens/s: 75514.9 | loss tokens/s: 57495.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.687253E-01 | lm-global-supervised-tokens: 1.300946E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.051748E-02 | lm-token-buffer-mib: 2.352827E+02 | global_load_balancing_loss: 1.001685E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11198.085 | params rms: 0.060634 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:08:51.640623 | finish at 2026-10-17 19:43:15 + [2026-10-13 11:38:07] iteration 728/ 2384 | consumed samples: 23296 | elapsed time per iteration (ms): 224437.4 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 74752.3 | valid tokens/s: 74752.3 | physical tokens/s: 75119.9 | loss tokens/s: 54467.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.286328E-01 | lm-global-supervised-tokens: 1.222454E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.187223E-02 | lm-token-buffer-mib: 2.336187E+02 | global_load_balancing_loss: 1.002169E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11198.033 | params rms: 0.060634 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:14:28.256395 | finish at 2026-10-17 18:52:36 + [2026-10-13 11:41:55] iteration 729/ 2384 | consumed samples: 23328 | elapsed time per iteration (ms): 227524.4 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 73738.1 | valid tokens/s: 73738.1 | physical tokens/s: 75084.3 | loss tokens/s: 55639.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.038746E-01 | lm-global-supervised-tokens: 1.265925E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.294890E-02 | lm-token-buffer-mib: 2.377928E+02 | global_load_balancing_loss: 1.001788E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11197.982 | params rms: 0.060633 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:35:52.937629 | finish at 2026-10-17 20:17:48 + [2026-10-13 11:46:01] iteration 730/ 2384 | consumed samples: 23360 | elapsed time per iteration (ms): 246313.5 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.21% | FLOPs estimator: workload-aware | logical tokens/s: 68113.1 | valid tokens/s: 68113.1 | physical tokens/s: 68655.8 | loss tokens/s: 51722.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.188626E-01 | lm-global-supervised-tokens: 1.273986E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.065022E-02 | lm-token-buffer-mib: 2.349288E+02 | global_load_balancing_loss: 1.000727E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11197.931 | params rms: 0.060633 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:10:02.540710 | finish at 2026-10-18 04:56:04 + [2026-10-13 11:49:42] iteration 731/ 2384 | consumed samples: 23392 | elapsed time per iteration (ms): 221011.4 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.13% | FLOPs estimator: workload-aware | logical tokens/s: 75910.9 | valid tokens/s: 75910.9 | physical tokens/s: 76902.6 | loss tokens/s: 60784.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.286929E-01 | lm-global-supervised-tokens: 1.343396E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.473890E-02 | lm-token-buffer-mib: 2.341722E+02 | global_load_balancing_loss: 9.987827E-01 | loss scale: 1.0 | grad norm: 0.218 | params norm: 11197.881 | params rms: 0.060633 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:28:51.830775 | finish at 2026-10-17 17:18:34 + [2026-10-13 11:53:28] iteration 732/ 2384 | consumed samples: 23424 | elapsed time per iteration (ms): 225463.1 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 74412.1 | valid tokens/s: 74412.1 | physical tokens/s: 75041.8 | loss tokens/s: 57702.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.412050E-01 | lm-global-supervised-tokens: 1.300986E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.164926E-02 | lm-token-buffer-mib: 2.353239E+02 | global_load_balancing_loss: 1.000020E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11197.833 | params rms: 0.060633 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:27:45.047824 | finish at 2026-10-17 19:21:13 + [2026-10-13 11:57:09] iteration 733/ 2384 | consumed samples: 23456 | elapsed time per iteration (ms): 221336.8 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 75799.3 | valid tokens/s: 75799.3 | physical tokens/s: 76384.7 | loss tokens/s: 55648.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.174643E-01 | lm-global-supervised-tokens: 1.231695E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.134514E-02 | lm-token-buffer-mib: 2.333158E+02 | global_load_balancing_loss: 9.994841E-01 | loss scale: 1.0 | grad norm: 0.291 | params norm: 11197.789 | params rms: 0.060632 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:30:27.100262 | finish at 2026-10-17 17:27:36 + [2026-10-13 12:00:57] iteration 734/ 2384 | consumed samples: 23488 | elapsed time per iteration (ms): 227701.8 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 73680.5 | valid tokens/s: 73680.5 | physical tokens/s: 74914.0 | loss tokens/s: 53343.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.363513E-01 | lm-global-supervised-tokens: 1.214649E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.471889E-02 | lm-token-buffer-mib: 2.362708E+02 | global_load_balancing_loss: 1.000077E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11197.740 | params rms: 0.060632 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:21:47.908809 | finish at 2026-10-17 20:22:45 + [2026-10-13 12:04:40] iteration 735/ 2384 | consumed samples: 23520 | elapsed time per iteration (ms): 223521.8 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 75058.4 | valid tokens/s: 75058.4 | physical tokens/s: 75640.9 | loss tokens/s: 52603.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.732439E-01 | lm-global-supervised-tokens: 1.175808E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.499839E-02 | lm-token-buffer-mib: 2.338167E+02 | global_load_balancing_loss: 1.000041E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11197.691 | params rms: 0.060632 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:23:07.476575 | finish at 2026-10-17 18:27:48 + [2026-10-13 12:08:24] iteration 736/ 2384 | consumed samples: 23552 | elapsed time per iteration (ms): 223567.2 | throughput per GPU (TFLOP/s/GPU): 26.0 | MFU 8.34% | FLOPs estimator: workload-aware | logical tokens/s: 75043.1 | valid tokens/s: 75043.1 | physical tokens/s: 75603.5 | loss tokens/s: 61166.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.765692E-01 | lm-global-supervised-tokens: 1.367485E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.100983E-02 | lm-token-buffer-mib: 2.337174E+02 | global_load_balancing_loss: 1.001401E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11197.645 | params rms: 0.060632 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:20:38.674000 | finish at 2026-10-17 18:29:03 + [2026-10-13 12:12:09] iteration 737/ 2384 | consumed samples: 23584 | elapsed time per iteration (ms): 224747.3 | throughput per GPU (TFLOP/s/GPU): 26.3 | MFU 8.44% | FLOPs estimator: workload-aware | logical tokens/s: 74649.1 | valid tokens/s: 74649.1 | physical tokens/s: 75158.4 | loss tokens/s: 50268.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.437567E-01 | lm-global-supervised-tokens: 1.129766E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.595661E-02 | lm-token-buffer-mib: 2.347937E+02 | global_load_balancing_loss: 1.001428E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11197.599 | params rms: 0.060631 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:49:18.875203 | finish at 2026-10-17 19:01:28 + [2026-10-13 12:15:53] iteration 738/ 2384 | consumed samples: 23616 | elapsed time per iteration (ms): 224707.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 74662.4 | valid tokens/s: 74662.3 | physical tokens/s: 75569.2 | loss tokens/s: 57563.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.757092E-01 | lm-global-supervised-tokens: 1.293494E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.398426E-02 | lm-token-buffer-mib: 2.346701E+02 | global_load_balancing_loss: 1.001111E+00 | loss scale: 1.0 | grad norm: 0.241 | params norm: 11197.549 | params rms: 0.060631 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:44:27.950034 | finish at 2026-10-17 19:00:21 + [2026-10-13 12:19:39] iteration 739/ 2384 | consumed samples: 23648 | elapsed time per iteration (ms): 225293.5 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 74468.1 | valid tokens/s: 74468.1 | physical tokens/s: 75579.0 | loss tokens/s: 54925.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.381570E-01 | lm-global-supervised-tokens: 1.237426E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.980169E-02 | lm-token-buffer-mib: 2.377553E+02 | global_load_balancing_loss: 9.998122E-01 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11197.503 | params rms: 0.060631 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:56:47.864673 | finish at 2026-10-17 19:16:26 + [2026-10-13 12:23:47] iteration 740/ 2384 | consumed samples: 23680 | elapsed time per iteration (ms): 248576.6 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 67493.0 | valid tokens/s: 67493.0 | physical tokens/s: 68666.7 | loss tokens/s: 50134.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.509976E-01 | lm-global-supervised-tokens: 1.246236E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.265719E-02 | lm-token-buffer-mib: 2.371351E+02 | global_load_balancing_loss: 1.000885E+00 | loss scale: 1.0 | grad norm: 0.319 | params norm: 11197.457 | params rms: 0.060631 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:30:59.885056 | finish at 2026-10-18 05:54:47 + [2026-10-13 12:27:34] iteration 741/ 2384 | consumed samples: 23712 | elapsed time per iteration (ms): 226415.6 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 74099.0 | valid tokens/s: 74099.0 | physical tokens/s: 74837.2 | loss tokens/s: 54438.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.338905E-01 | lm-global-supervised-tokens: 1.232581E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.586886E-02 | lm-token-buffer-mib: 2.356082E+02 | global_load_balancing_loss: 1.000672E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11197.413 | params rms: 0.060630 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:20:00.883392 | finish at 2026-10-17 19:47:35 + [2026-10-13 12:31:18] iteration 742/ 2384 | consumed samples: 23744 | elapsed time per iteration (ms): 224499.5 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 74731.5 | valid tokens/s: 74731.5 | physical tokens/s: 75564.4 | loss tokens/s: 55099.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.369270E-01 | lm-global-supervised-tokens: 1.236970E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.502270E-02 | lm-token-buffer-mib: 2.341598E+02 | global_load_balancing_loss: 1.000176E+00 | loss scale: 1.0 | grad norm: 0.205 | params norm: 11197.369 | params rms: 0.060630 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:23:48.200200 | finish at 2026-10-17 18:55:06 + [2026-10-13 12:35:06] iteration 743/ 2384 | consumed samples: 23776 | elapsed time per iteration (ms): 227523.6 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 73738.4 | valid tokens/s: 73738.4 | physical tokens/s: 74725.4 | loss tokens/s: 52631.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.442865E-01 | lm-global-supervised-tokens: 1.197499E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.264757E-02 | lm-token-buffer-mib: 2.369137E+02 | global_load_balancing_loss: 1.001190E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11197.324 | params rms: 0.060630 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:42:46.203118 | finish at 2026-10-17 20:17:52 + [2026-10-13 12:38:46] iteration 744/ 2384 | consumed samples: 23808 | elapsed time per iteration (ms): 219969.7 | throughput per GPU (TFLOP/s/GPU): 18.0 | MFU 5.77% | FLOPs estimator: workload-aware | logical tokens/s: 76270.6 | valid tokens/s: 76270.6 | physical tokens/s: 77233.3 | loss tokens/s: 60900.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.986225E-01 | lm-global-supervised-tokens: 1.339621E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.522762E-02 | lm-token-buffer-mib: 2.337983E+02 | global_load_balancing_loss: 1.000071E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11197.282 | params rms: 0.060630 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:12:30.235434 | finish at 2026-10-17 16:51:16 + [2026-10-13 12:42:34] iteration 745/ 2384 | consumed samples: 23840 | elapsed time per iteration (ms): 227921.1 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 73609.7 | valid tokens/s: 73609.7 | physical tokens/s: 74668.9 | loss tokens/s: 55525.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.064725E-01 | lm-global-supervised-tokens: 1.265552E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.961523E-02 | lm-token-buffer-mib: 2.366193E+02 | global_load_balancing_loss: 9.994727E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11197.242 | params rms: 0.060629 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:46:02.736273 | finish at 2026-10-17 20:28:36 + [2026-10-13 12:46:14] iteration 746/ 2384 | consumed samples: 23872 | elapsed time per iteration (ms): 220009.2 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.67% | FLOPs estimator: workload-aware | logical tokens/s: 76256.7 | valid tokens/s: 76256.7 | physical tokens/s: 76816.6 | loss tokens/s: 58825.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.767231E-01 | lm-global-supervised-tokens: 1.294215E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.324082E-02 | lm-token-buffer-mib: 2.328476E+02 | global_load_balancing_loss: 1.000100E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11197.202 | params rms: 0.060629 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:06:15.015083 | finish at 2026-10-17 16:52:29 + [2026-10-13 12:49:57] iteration 747/ 2384 | consumed samples: 23904 | elapsed time per iteration (ms): 223032.3 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75223.2 | valid tokens/s: 75223.2 | physical tokens/s: 75839.9 | loss tokens/s: 58311.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.540918E-01 | lm-global-supervised-tokens: 1.300536E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.563115E-02 | lm-token-buffer-mib: 2.329986E+02 | global_load_balancing_loss: 1.000106E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11197.164 | params rms: 0.060629 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:25:03.796644 | finish at 2026-10-17 18:15:00 + [2026-10-13 12:53:42] iteration 748/ 2384 | consumed samples: 23936 | elapsed time per iteration (ms): 225340.4 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.95% | FLOPs estimator: workload-aware | logical tokens/s: 74452.5 | valid tokens/s: 74452.5 | physical tokens/s: 74765.5 | loss tokens/s: 55870.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.263192E-01 | lm-global-supervised-tokens: 1.258979E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.211366E-02 | lm-token-buffer-mib: 2.332303E+02 | global_load_balancing_loss: 1.000446E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11197.124 | params rms: 0.060629 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:24:16.893978 | finish at 2026-10-17 19:17:59 + [2026-10-13 12:57:22] iteration 749/ 2384 | consumed samples: 23968 | elapsed time per iteration (ms): 220244.0 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 76175.6 | valid tokens/s: 76175.6 | physical tokens/s: 76630.6 | loss tokens/s: 56342.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.777633E-01 | lm-global-supervised-tokens: 1.240917E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.222098E-02 | lm-token-buffer-mib: 2.331569E+02 | global_load_balancing_loss: 1.000739E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11197.079 | params rms: 0.060629 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:01:38.962930 | finish at 2026-10-17 16:59:01 + [2026-10-13 13:01:25] iteration 750/ 2384 | consumed samples: 24000 | elapsed time per iteration (ms): 243252.7 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 68970.2 | valid tokens/s: 68970.2 | physical tokens/s: 69417.6 | loss tokens/s: 54068.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.592895E-01 | lm-global-supervised-tokens: 1.315225E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.270236E-02 | lm-token-buffer-mib: 2.338841E+02 | global_load_balancing_loss: 1.000544E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11197.036 | params rms: 0.060628 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:24:34.984409 | finish at 2026-10-18 03:26:00 + [2026-10-13 13:05:09] iteration 751/ 2384 | consumed samples: 24032 | elapsed time per iteration (ms): 223104.9 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75198.7 | valid tokens/s: 75198.7 | physical tokens/s: 76996.5 | loss tokens/s: 58383.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.668917E-01 | lm-global-supervised-tokens: 1.302558E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.865866E-02 | lm-token-buffer-mib: 2.350897E+02 | global_load_balancing_loss: 1.000197E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11196.998 | params rms: 0.060628 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:12:10.380156 | finish at 2026-10-17 18:17:19 + [2026-10-13 13:09:34] iteration 752/ 2384 | consumed samples: 24064 | elapsed time per iteration (ms): 265194.9 | throughput per GPU (TFLOP/s/GPU): 17.9 | MFU 5.75% | FLOPs estimator: workload-aware | logical tokens/s: 63263.7 | valid tokens/s: 63263.7 | physical tokens/s: 64878.7 | loss tokens/s: 48533.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.181992E-01 | lm-global-supervised-tokens: 1.287079E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.247195E-02 | lm-token-buffer-mib: 2.373016E+02 | global_load_balancing_loss: 1.000388E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11196.958 | params rms: 0.060628 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:13:18.109543 | finish at 2026-10-18 13:22:52 + [2026-10-13 13:13:19] iteration 753/ 2384 | consumed samples: 24096 | elapsed time per iteration (ms): 225031.3 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 74554.9 | valid tokens/s: 74554.9 | physical tokens/s: 75859.0 | loss tokens/s: 59078.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.619164E-01 | lm-global-supervised-tokens: 1.329455E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.505944E-02 | lm-token-buffer-mib: 2.377166E+02 | global_load_balancing_loss: 9.996926E-01 | loss scale: 1.0 | grad norm: 0.182 | params norm: 11196.916 | params rms: 0.060628 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:57:06.088519 | finish at 2026-10-17 19:10:25 + [2026-10-13 13:17:10] iteration 754/ 2384 | consumed samples: 24128 | elapsed time per iteration (ms): 230986.7 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 72632.7 | valid tokens/s: 72632.7 | physical tokens/s: 73007.4 | loss tokens/s: 52200.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.655586E-01 | lm-global-supervised-tokens: 1.205771E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.507682E-02 | lm-token-buffer-mib: 2.331926E+02 | global_load_balancing_loss: 1.001186E+00 | loss scale: 1.0 | grad norm: 0.185 | params norm: 11196.873 | params rms: 0.060627 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:35:08.259692 | finish at 2026-10-17 21:52:18 + [2026-10-13 13:20:52] iteration 755/ 2384 | consumed samples: 24160 | elapsed time per iteration (ms): 222421.7 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.63% | FLOPs estimator: workload-aware | logical tokens/s: 75429.7 | valid tokens/s: 75429.7 | physical tokens/s: 75967.6 | loss tokens/s: 60390.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.259463E-01 | lm-global-supervised-tokens: 1.343223E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.454423E-02 | lm-token-buffer-mib: 2.323969E+02 | global_load_balancing_loss: 1.000505E+00 | loss scale: 1.0 | grad norm: 0.267 | params norm: 11196.829 | params rms: 0.060627 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:38:44.905802 | finish at 2026-10-17 17:59:37 + [2026-10-13 13:24:35] iteration 756/ 2384 | consumed samples: 24192 | elapsed time per iteration (ms): 222991.8 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 75236.8 | valid tokens/s: 75236.8 | physical tokens/s: 75716.2 | loss tokens/s: 55066.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.411295E-01 | lm-global-supervised-tokens: 1.227941E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.658393E-02 | lm-token-buffer-mib: 2.334571E+02 | global_load_balancing_loss: 9.997580E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11196.782 | params rms: 0.060627 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:50:30.593068 | finish at 2026-10-17 18:15:06 + [2026-10-13 13:28:17] iteration 757/ 2384 | consumed samples: 24224 | elapsed time per iteration (ms): 221888.3 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75610.9 | valid tokens/s: 75610.9 | physical tokens/s: 76188.8 | loss tokens/s: 56970.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.393040E-01 | lm-global-supervised-tokens: 1.264118E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.524550E-02 | lm-token-buffer-mib: 2.343345E+02 | global_load_balancing_loss: 9.992833E-01 | loss scale: 1.0 | grad norm: 0.268 | params norm: 11196.739 | params rms: 0.060627 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:16:52.199613 | finish at 2026-10-17 17:45:09 + [2026-10-13 13:31:57] iteration 758/ 2384 | consumed samples: 24256 | elapsed time per iteration (ms): 219978.5 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 76267.5 | valid tokens/s: 76267.5 | physical tokens/s: 77005.1 | loss tokens/s: 56526.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.109710E-01 | lm-global-supervised-tokens: 1.243453E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.039978E-02 | lm-token-buffer-mib: 2.349515E+02 | global_load_balancing_loss: 1.000049E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11196.697 | params rms: 0.060626 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:21:24.975304 | finish at 2026-10-17 16:53:22 + [2026-10-13 13:35:43] iteration 759/ 2384 | consumed samples: 24288 | elapsed time per iteration (ms): 226053.8 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 74217.8 | valid tokens/s: 74217.7 | physical tokens/s: 75000.6 | loss tokens/s: 57454.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.673053E-01 | lm-global-supervised-tokens: 1.298776E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.149031E-02 | lm-token-buffer-mib: 2.351239E+02 | global_load_balancing_loss: 1.000857E+00 | loss scale: 1.0 | grad norm: 0.339 | params norm: 11196.656 | params rms: 0.060626 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:02:17.479413 | finish at 2026-10-17 19:38:01 + [2026-10-13 13:39:49] iteration 760/ 2384 | consumed samples: 24320 | elapsed time per iteration (ms): 245784.5 | throughput per GPU (TFLOP/s/GPU): 15.7 | MFU 5.03% | FLOPs estimator: workload-aware | logical tokens/s: 68259.8 | valid tokens/s: 68259.8 | physical tokens/s: 69140.8 | loss tokens/s: 51383.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.264051E-01 | lm-global-supervised-tokens: 1.262916E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.159384E-02 | lm-token-buffer-mib: 2.360955E+02 | global_load_balancing_loss: 1.000862E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11196.617 | params rms: 0.060626 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:52:34.094021 | finish at 2026-10-18 04:32:23 +times across ranks (ms): + save-checkpoint: + rank 0: 24100.20 + rank 1: 24100.22 + rank 2: 24100.23 + rank 3: 24100.24 + rank 4: 24100.21 + rank 5: 24100.22 + rank 6: 24100.29 + rank 7: 24100.21 + rank 8: 24099.53 + rank 9: 24099.54 + rank 10: 24099.54 + rank 11: 24099.55 + rank 12: 24099.52 + rank 13: 24099.52 + rank 14: 24099.52 + rank 15: 24099.52 + rank 16: 24099.67 + rank 17: 24099.66 + rank 18: 24099.66 + rank 19: 24099.69 + rank 20: 24099.66 + rank 21: 24099.67 + rank 22: 24099.65 + rank 23: 24099.68 + rank 24: 24099.79 + rank 25: 24099.80 + rank 26: 24099.79 + rank 27: 24099.81 + rank 28: 24099.80 + rank 29: 24099.87 + rank 30: 24099.82 + rank 31: 24099.79 + rank 32: 24098.63 + rank 33: 24098.65 + rank 34: 24098.69 + rank 35: 24098.69 + rank 36: 24098.65 + rank 37: 24098.69 + rank 38: 24098.65 + rank 39: 24098.67 + rank 40: 24100.98 + rank 41: 24100.97 + rank 42: 24100.94 + rank 43: 24100.97 + rank 44: 24100.95 + rank 45: 24100.97 + rank 46: 24100.96 + rank 47: 24100.93 + rank 48: 24099.60 + rank 49: 24099.59 + rank 50: 24099.61 + rank 51: 24099.60 + rank 52: 24099.59 + rank 53: 24099.61 + rank 54: 24099.61 + rank 55: 24099.59 + rank 56: 24099.34 + rank 57: 24099.34 + rank 58: 24099.36 + rank 59: 24099.33 + rank 60: 24099.34 + rank 61: 24099.42 + rank 62: 24099.35 + rank 63: 24099.38 + [2026-10-13 13:43:58] iteration 761/ 2384 | consumed samples: 24352 | elapsed time per iteration (ms): 224935.0 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 74586.9 | valid tokens/s: 74586.9 | physical tokens/s: 75333.0 | loss tokens/s: 62171.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 9.367137E-01 | lm-global-supervised-tokens: 1.398459E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.308047E-01 | lm-token-buffer-mib: 2.345852E+02 | global_load_balancing_loss: 1.000834E+00 | loss scale: 1.0 | grad norm: 0.229 | params norm: 11196.582 | params rms: 0.060626 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:24:29.424421 | finish at 2026-10-17 19:08:27 + [2026-10-13 13:47:48] iteration 762/ 2384 | consumed samples: 24384 | elapsed time per iteration (ms): 229764.3 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 73019.1 | valid tokens/s: 73019.1 | physical tokens/s: 74766.9 | loss tokens/s: 57302.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.682671E-01 | lm-global-supervised-tokens: 1.316608E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.431967E-02 | lm-token-buffer-mib: 2.346323E+02 | global_load_balancing_loss: 1.001042E+00 | loss scale: 1.0 | grad norm: 0.355 | params norm: 11196.547 | params rms: 0.060626 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:31:17.639088 | finish at 2026-10-17 21:19:05 + [2026-10-13 13:51:28] iteration 763/ 2384 | consumed samples: 24416 | elapsed time per iteration (ms): 220530.4 | throughput per GPU (TFLOP/s/GPU): 17.7 | MFU 5.68% | FLOPs estimator: workload-aware | logical tokens/s: 76076.5 | valid tokens/s: 76076.5 | physical tokens/s: 77110.4 | loss tokens/s: 58416.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.181224E-01 | lm-global-supervised-tokens: 1.288250E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.882519E-02 | lm-token-buffer-mib: 2.348741E+02 | global_load_balancing_loss: 1.000718E+00 | loss scale: 1.0 | grad norm: 0.453 | params norm: 11196.513 | params rms: 0.060625 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:17:59.852278 | finish at 2026-10-17 17:09:28 + [2026-10-13 13:55:09] iteration 764/ 2384 | consumed samples: 24448 | elapsed time per iteration (ms): 220798.1 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 75984.3 | valid tokens/s: 75984.3 | physical tokens/s: 76640.4 | loss tokens/s: 58457.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.804708E-01 | lm-global-supervised-tokens: 1.290730E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.314189E-02 | lm-token-buffer-mib: 2.335327E+02 | global_load_balancing_loss: 1.001095E+00 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11196.478 | params rms: 0.060625 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:21:32.938213 | finish at 2026-10-17 17:16:42 + [2026-10-13 13:58:53] iteration 765/ 2384 | consumed samples: 24480 | elapsed time per iteration (ms): 224189.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 74834.9 | valid tokens/s: 74834.9 | physical tokens/s: 75936.1 | loss tokens/s: 55416.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.814953E-01 | lm-global-supervised-tokens: 1.242388E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.159647E-02 | lm-token-buffer-mib: 2.372473E+02 | global_load_balancing_loss: 1.000929E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11196.443 | params rms: 0.060625 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:49:22.755875 | finish at 2026-10-17 18:48:16 + [2026-10-13 14:02:36] iteration 766/ 2384 | consumed samples: 24512 | elapsed time per iteration (ms): 222832.6 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 75290.6 | valid tokens/s: 75290.6 | physical tokens/s: 76125.3 | loss tokens/s: 58401.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.405703E-01 | lm-global-supervised-tokens: 1.301375E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.265416E-02 | lm-token-buffer-mib: 2.335553E+02 | global_load_balancing_loss: 1.000816E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11196.407 | params rms: 0.060625 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:09:03.173992 | finish at 2026-10-17 18:11:39 + [2026-10-13 14:06:20] iteration 767/ 2384 | consumed samples: 24544 | elapsed time per iteration (ms): 224338.4 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.76% | FLOPs estimator: workload-aware | logical tokens/s: 74785.2 | valid tokens/s: 74785.2 | physical tokens/s: 75553.6 | loss tokens/s: 58712.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.290405E-01 | lm-global-supervised-tokens: 1.317137E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.242228E-02 | lm-token-buffer-mib: 2.338105E+02 | global_load_balancing_loss: 9.992988E-01 | loss scale: 1.0 | grad norm: 0.231 | params norm: 11196.370 | params rms: 0.060625 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:45:55.162547 | finish at 2026-10-17 18:52:16 + [2026-10-13 14:10:03] iteration 768/ 2384 | consumed samples: 24576 | elapsed time per iteration (ms): 222137.0 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75526.3 | valid tokens/s: 75526.3 | physical tokens/s: 76045.3 | loss tokens/s: 57574.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.472900E-01 | lm-global-supervised-tokens: 1.278940E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.648221E-02 | lm-token-buffer-mib: 2.334325E+02 | global_load_balancing_loss: 9.999006E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11196.330 | params rms: 0.060624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:42:53.444534 | finish at 2026-10-17 17:52:56 + [2026-10-13 14:13:45] iteration 769/ 2384 | consumed samples: 24608 | elapsed time per iteration (ms): 222182.9 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 75510.7 | valid tokens/s: 75510.7 | physical tokens/s: 76481.9 | loss tokens/s: 59079.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.625352E-01 | lm-global-supervised-tokens: 1.312653E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.313573E-02 | lm-token-buffer-mib: 2.354777E+02 | global_load_balancing_loss: 9.994410E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11196.291 | params rms: 0.060624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:40:25.340682 | finish at 2026-10-17 17:54:10 + [2026-10-13 14:17:48] iteration 770/ 2384 | consumed samples: 24640 | elapsed time per iteration (ms): 243196.8 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 68986.0 | valid tokens/s: 68986.0 | physical tokens/s: 69938.6 | loss tokens/s: 53673.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.377149E-01 | lm-global-supervised-tokens: 1.305321E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.454965E-02 | lm-token-buffer-mib: 2.348386E+02 | global_load_balancing_loss: 1.000072E+00 | loss scale: 1.0 | grad norm: 0.261 | params norm: 11196.253 | params rms: 0.060624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:01:59.600557 | finish at 2026-10-18 03:19:47 + [2026-10-13 14:21:34] iteration 771/ 2384 | consumed samples: 24672 | elapsed time per iteration (ms): 226330.5 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 74126.9 | valid tokens/s: 74126.9 | physical tokens/s: 75030.8 | loss tokens/s: 59695.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.904686E-01 | lm-global-supervised-tokens: 1.351093E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.455163E-02 | lm-token-buffer-mib: 2.351065E+02 | global_load_balancing_loss: 1.000624E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11196.217 | params rms: 0.060624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:24:31.172848 | finish at 2026-10-17 19:46:05 + [2026-10-13 14:25:18] iteration 772/ 2384 | consumed samples: 24704 | elapsed time per iteration (ms): 223488.9 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 75069.4 | valid tokens/s: 75069.4 | physical tokens/s: 75920.7 | loss tokens/s: 56454.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.369939E-01 | lm-global-supervised-tokens: 1.261700E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.120636E-02 | lm-token-buffer-mib: 2.355519E+02 | global_load_balancing_loss: 1.001468E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11196.183 | params rms: 0.060624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:04:24.049064 | finish at 2026-10-17 18:29:42 + [2026-10-13 14:28:59] iteration 773/ 2384 | consumed samples: 24736 | elapsed time per iteration (ms): 221742.0 | throughput per GPU (TFLOP/s/GPU): 27.1 | MFU 8.69% | FLOPs estimator: workload-aware | logical tokens/s: 75660.8 | valid tokens/s: 75660.8 | physical tokens/s: 76377.6 | loss tokens/s: 55242.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.512100E-01 | lm-global-supervised-tokens: 1.224949E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.830237E-02 | lm-token-buffer-mib: 2.324420E+02 | global_load_balancing_loss: 1.000929E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11196.150 | params rms: 0.060624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:13:46.420932 | finish at 2026-10-17 17:42:46 + [2026-10-13 14:32:42] iteration 774/ 2384 | consumed samples: 24768 | elapsed time per iteration (ms): 222458.7 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 75417.1 | valid tokens/s: 75417.1 | physical tokens/s: 75988.0 | loss tokens/s: 59237.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.800561E-01 | lm-global-supervised-tokens: 1.317791E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.119737E-02 | lm-token-buffer-mib: 2.332684E+02 | global_load_balancing_loss: 1.002920E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11196.112 | params rms: 0.060623 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:29:18.493087 | finish at 2026-10-17 18:02:00 + [2026-10-13 14:36:28] iteration 775/ 2384 | consumed samples: 24800 | elapsed time per iteration (ms): 226141.7 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 74188.9 | valid tokens/s: 74188.9 | physical tokens/s: 74667.8 | loss tokens/s: 56367.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.644087E-01 | lm-global-supervised-tokens: 1.274698E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.448319E-02 | lm-token-buffer-mib: 2.343685E+02 | global_load_balancing_loss: 1.000613E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11196.075 | params rms: 0.060623 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:04:21.973481 | finish at 2026-10-17 19:40:50 + [2026-10-13 14:40:10] iteration 776/ 2384 | consumed samples: 24832 | elapsed time per iteration (ms): 221830.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75630.7 | valid tokens/s: 75630.7 | physical tokens/s: 76330.4 | loss tokens/s: 59325.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.838338E-01 | lm-global-supervised-tokens: 1.316014E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.361057E-02 | lm-token-buffer-mib: 2.336073E+02 | global_load_balancing_loss: 1.000361E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11196.040 | params rms: 0.060623 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:05:03.490591 | finish at 2026-10-17 17:45:13 + [2026-10-13 14:43:55] iteration 777/ 2384 | consumed samples: 24864 | elapsed time per iteration (ms): 224875.9 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 74606.4 | valid tokens/s: 74606.4 | physical tokens/s: 75574.2 | loss tokens/s: 56799.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.837265E-01 | lm-global-supervised-tokens: 1.277290E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.793424E-02 | lm-token-buffer-mib: 2.349457E+02 | global_load_balancing_loss: 9.990916E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11196.004 | params rms: 0.060623 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:22:55.640697 | finish at 2026-10-17 19:06:50 + [2026-10-13 14:47:36] iteration 778/ 2384 | consumed samples: 24896 | elapsed time per iteration (ms): 220824.4 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 75975.2 | valid tokens/s: 75975.2 | physical tokens/s: 76514.7 | loss tokens/s: 59145.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.083848E-01 | lm-global-supervised-tokens: 1.306071E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.287268E-02 | lm-token-buffer-mib: 2.324814E+02 | global_load_balancing_loss: 9.991760E-01 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11195.967 | params rms: 0.060623 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:30:43.917395 | finish at 2026-10-17 17:18:20 + [2026-10-13 14:51:15] iteration 779/ 2384 | consumed samples: 24928 | elapsed time per iteration (ms): 219198.6 | throughput per GPU (TFLOP/s/GPU): 17.3 | MFU 5.56% | FLOPs estimator: workload-aware | logical tokens/s: 76538.9 | valid tokens/s: 76538.9 | physical tokens/s: 77283.1 | loss tokens/s: 57788.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.221339E-01 | lm-global-supervised-tokens: 1.266722E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.321759E-02 | lm-token-buffer-mib: 2.335692E+02 | global_load_balancing_loss: 9.981020E-01 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11195.930 | params rms: 0.060622 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:43:33.776429 | finish at 2026-10-17 16:34:49 + [2026-10-13 14:55:17] iteration 780/ 2384 | consumed samples: 24960 | elapsed time per iteration (ms): 242114.1 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 69294.6 | valid tokens/s: 69294.6 | physical tokens/s: 69680.9 | loss tokens/s: 52832.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.174286E-01 | lm-global-supervised-tokens: 1.279159E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.796899E-02 | lm-token-buffer-mib: 2.324894E+02 | global_load_balancing_loss: 9.999367E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11195.889 | params rms: 0.060622 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:52:31.026692 | finish at 2026-10-18 02:47:48 + [2026-10-13 14:59:04] iteration 781/ 2384 | consumed samples: 24992 | elapsed time per iteration (ms): 227211.0 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 73839.7 | valid tokens/s: 73839.7 | physical tokens/s: 74420.9 | loss tokens/s: 54654.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.185862E-01 | lm-global-supervised-tokens: 1.241817E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.240866E-02 | lm-token-buffer-mib: 2.342642E+02 | global_load_balancing_loss: 1.001744E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11195.846 | params rms: 0.060622 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:10:19.292566 | finish at 2026-10-17 20:09:23 + [2026-10-13 15:02:49] iteration 782/ 2384 | consumed samples: 25024 | elapsed time per iteration (ms): 225298.7 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 74466.4 | valid tokens/s: 74466.4 | physical tokens/s: 75284.7 | loss tokens/s: 56188.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.101963E-01 | lm-global-supervised-tokens: 1.265920E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.249714E-02 | lm-token-buffer-mib: 2.360652E+02 | global_load_balancing_loss: 1.001254E+00 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11195.804 | params rms: 0.060622 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:15:28.443835 | finish at 2026-10-17 19:18:18 + [2026-10-13 15:06:31] iteration 783/ 2384 | consumed samples: 25056 | elapsed time per iteration (ms): 221464.9 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.83% | FLOPs estimator: workload-aware | logical tokens/s: 75755.5 | valid tokens/s: 75755.4 | physical tokens/s: 76897.4 | loss tokens/s: 58129.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.451791E-01 | lm-global-supervised-tokens: 1.287355E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.403373E-02 | lm-token-buffer-mib: 2.327535E+02 | global_load_balancing_loss: 1.002255E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11195.764 | params rms: 0.060621 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:29:25.330883 | finish at 2026-10-17 17:35:56 + [2026-10-13 15:10:14] iteration 784/ 2384 | consumed samples: 25088 | elapsed time per iteration (ms): 222743.8 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 75320.5 | valid tokens/s: 75320.5 | physical tokens/s: 76023.8 | loss tokens/s: 53895.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.795786E-01 | lm-global-supervised-tokens: 1.200477E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.302028E-02 | lm-token-buffer-mib: 2.341269E+02 | global_load_balancing_loss: 9.996227E-01 | loss scale: 1.0 | grad norm: 0.306 | params norm: 11195.720 | params rms: 0.060621 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:59:50.073776 | finish at 2026-10-17 18:10:04 + [2026-10-13 15:13:59] iteration 785/ 2384 | consumed samples: 25120 | elapsed time per iteration (ms): 225740.7 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.65% | FLOPs estimator: workload-aware | logical tokens/s: 74320.7 | valid tokens/s: 74320.7 | physical tokens/s: 74951.3 | loss tokens/s: 58337.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.994999E-01 | lm-global-supervised-tokens: 1.316920E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.443559E-02 | lm-token-buffer-mib: 2.348113E+02 | global_load_balancing_loss: 1.000769E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11195.675 | params rms: 0.060621 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:15:59.341568 | finish at 2026-10-17 19:29:59 + [2026-10-13 15:17:45] iteration 786/ 2384 | consumed samples: 25152 | elapsed time per iteration (ms): 225152.5 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 74514.8 | valid tokens/s: 74514.8 | physical tokens/s: 75187.7 | loss tokens/s: 56717.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.226964E-01 | lm-global-supervised-tokens: 1.276996E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.597932E-02 | lm-token-buffer-mib: 2.346754E+02 | global_load_balancing_loss: 1.001115E+00 | loss scale: 1.0 | grad norm: 0.257 | params norm: 11195.633 | params rms: 0.060621 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:56:33.719627 | finish at 2026-10-17 19:14:18 + [2026-10-13 15:21:29] iteration 787/ 2384 | consumed samples: 25184 | elapsed time per iteration (ms): 224312.7 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.06% | FLOPs estimator: workload-aware | logical tokens/s: 74793.6 | valid tokens/s: 74793.6 | physical tokens/s: 75424.7 | loss tokens/s: 53623.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.810912E-01 | lm-global-supervised-tokens: 1.202848E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.383741E-02 | lm-token-buffer-mib: 2.343040E+02 | global_load_balancing_loss: 9.992850E-01 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11195.593 | params rms: 0.060620 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:30:27.402133 | finish at 2026-10-17 18:51:56 + [2026-10-13 15:25:15] iteration 788/ 2384 | consumed samples: 25216 | elapsed time per iteration (ms): 226371.4 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 74113.5 | valid tokens/s: 74113.5 | physical tokens/s: 74847.4 | loss tokens/s: 59562.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.282441E-01 | lm-global-supervised-tokens: 1.348321E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.050397E-02 | lm-token-buffer-mib: 2.349936E+02 | global_load_balancing_loss: 9.990958E-01 | loss scale: 1.0 | grad norm: 0.271 | params norm: 11195.553 | params rms: 0.060620 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:21:28.675441 | finish at 2026-10-17 19:46:44 + [2026-10-13 15:28:56] iteration 789/ 2384 | consumed samples: 25248 | elapsed time per iteration (ms): 220940.8 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 75935.2 | valid tokens/s: 75935.2 | physical tokens/s: 76956.4 | loss tokens/s: 56454.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.492123E-01 | lm-global-supervised-tokens: 1.247317E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.472492E-02 | lm-token-buffer-mib: 2.351272E+02 | global_load_balancing_loss: 9.994737E-01 | loss scale: 1.0 | grad norm: 0.330 | params norm: 11195.512 | params rms: 0.060620 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:53:20.551965 | finish at 2026-10-17 17:22:17 + [2026-10-13 15:33:01] iteration 790/ 2384 | consumed samples: 25280 | elapsed time per iteration (ms): 244619.4 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 68584.9 | valid tokens/s: 68584.9 | physical tokens/s: 69014.3 | loss tokens/s: 47133.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.043543E-01 | lm-global-supervised-tokens: 1.152980E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.370977E-02 | lm-token-buffer-mib: 2.342147E+02 | global_load_balancing_loss: 9.998334E-01 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11195.469 | params rms: 0.060620 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:18:43.286775 | finish at 2026-10-18 03:51:44 + [2026-10-13 15:36:47] iteration 791/ 2384 | consumed samples: 25312 | elapsed time per iteration (ms): 226666.3 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 74017.2 | valid tokens/s: 74017.2 | physical tokens/s: 75339.2 | loss tokens/s: 58506.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.370547E-01 | lm-global-supervised-tokens: 1.326133E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.683582E-02 | lm-token-buffer-mib: 2.362511E+02 | global_load_balancing_loss: 1.001021E+00 | loss scale: 1.0 | grad norm: 0.378 | params norm: 11195.427 | params rms: 0.060620 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:17:59.339273 | finish at 2026-10-17 19:54:47 + [2026-10-13 15:40:29] iteration 792/ 2384 | consumed samples: 25344 | elapsed time per iteration (ms): 221500.7 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75743.1 | valid tokens/s: 75743.1 | physical tokens/s: 76583.9 | loss tokens/s: 55698.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.248490E-01 | lm-global-supervised-tokens: 1.233730E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.371003E-02 | lm-token-buffer-mib: 2.331630E+02 | global_load_balancing_loss: 1.000845E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11195.388 | params rms: 0.060619 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:57:09.128059 | finish at 2026-10-17 17:37:38 + [2026-10-13 15:44:11] iteration 793/ 2384 | consumed samples: 25376 | elapsed time per iteration (ms): 221678.3 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75682.5 | valid tokens/s: 75682.5 | physical tokens/s: 76177.3 | loss tokens/s: 58430.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.751657E-01 | lm-global-supervised-tokens: 1.295270E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.321280E-02 | lm-token-buffer-mib: 2.337389E+02 | global_load_balancing_loss: 1.001062E+00 | loss scale: 1.0 | grad norm: 0.282 | params norm: 11195.345 | params rms: 0.060619 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:58:10.235080 | finish at 2026-10-17 17:42:21 + [2026-10-13 15:47:51] iteration 794/ 2384 | consumed samples: 25408 | elapsed time per iteration (ms): 219913.1 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 76290.1 | valid tokens/s: 76290.1 | physical tokens/s: 76768.6 | loss tokens/s: 56295.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.375203E-01 | lm-global-supervised-tokens: 1.238015E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.916904E-02 | lm-token-buffer-mib: 2.327134E+02 | global_load_balancing_loss: 1.002748E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11195.304 | params rms: 0.060619 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:07:41.841516 | finish at 2026-10-17 16:55:32 + [2026-10-13 15:51:36] iteration 795/ 2384 | consumed samples: 25440 | elapsed time per iteration (ms): 225260.4 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 74478.9 | valid tokens/s: 74478.9 | physical tokens/s: 74792.9 | loss tokens/s: 53077.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.548140E-01 | lm-global-supervised-tokens: 1.195633E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.199105E-02 | lm-token-buffer-mib: 2.329698E+02 | global_load_balancing_loss: 1.001302E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11195.262 | params rms: 0.060619 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:25:38.776448 | finish at 2026-10-17 19:17:15 + [2026-10-13 15:55:16] iteration 796/ 2384 | consumed samples: 25472 | elapsed time per iteration (ms): 219747.5 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 76347.6 | valid tokens/s: 76347.6 | physical tokens/s: 76668.7 | loss tokens/s: 57794.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.083328E-01 | lm-global-supervised-tokens: 1.270028E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.257805E-02 | lm-token-buffer-mib: 2.324185E+02 | global_load_balancing_loss: 9.991595E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11195.215 | params rms: 0.060618 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:55:58.973875 | finish at 2026-10-17 16:51:15 + [2026-10-13 15:58:59] iteration 797/ 2384 | consumed samples: 25504 | elapsed time per iteration (ms): 223842.1 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.78% | FLOPs estimator: workload-aware | logical tokens/s: 74951.0 | valid tokens/s: 74951.0 | physical tokens/s: 75358.8 | loss tokens/s: 58186.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.864247E-01 | lm-global-supervised-tokens: 1.302468E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.588957E-02 | lm-token-buffer-mib: 2.341437E+02 | global_load_balancing_loss: 9.982485E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11195.170 | params rms: 0.060618 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:40:37.345956 | finish at 2026-10-17 18:39:37 + [2026-10-13 16:02:41] iteration 798/ 2384 | consumed samples: 25536 | elapsed time per iteration (ms): 221185.2 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 75851.2 | valid tokens/s: 75851.2 | physical tokens/s: 76666.8 | loss tokens/s: 58685.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.082892E-01 | lm-global-supervised-tokens: 1.298042E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.470509E-02 | lm-token-buffer-mib: 2.338080E+02 | global_load_balancing_loss: 1.000587E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11195.127 | params rms: 0.060618 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:26:39.768000 | finish at 2026-10-17 17:29:20 + [2026-10-13 16:06:19] iteration 799/ 2384 | consumed samples: 25568 | elapsed time per iteration (ms): 218236.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 76876.3 | valid tokens/s: 76876.3 | physical tokens/s: 77448.4 | loss tokens/s: 56259.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.864546E-01 | lm-global-supervised-tokens: 1.227780E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.072481E-02 | lm-token-buffer-mib: 2.328205E+02 | global_load_balancing_loss: 1.001183E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11195.087 | params rms: 0.060618 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:05:04.920938 | finish at 2026-10-17 16:11:24 + [2026-10-13 16:10:24] iteration 800/ 2384 | consumed samples: 25600 | elapsed time per iteration (ms): 245691.8 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 68285.5 | valid tokens/s: 68285.5 | physical tokens/s: 68871.7 | loss tokens/s: 51981.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.587285E-01 | lm-global-supervised-tokens: 1.277141E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.237514E-02 | lm-token-buffer-mib: 2.345406E+02 | global_load_balancing_loss: 1.001170E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11195.047 | params rms: 0.060618 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:06:15.860104 | finish at 2026-10-18 04:16:40 +times across ranks (ms): + save-checkpoint: + rank 0: 26201.20 + rank 1: 26201.20 + rank 2: 26201.20 + rank 3: 26201.20 + rank 4: 26201.19 + rank 5: 26201.23 + rank 6: 26201.22 + rank 7: 26201.19 + rank 8: 26200.98 + rank 9: 26201.02 + rank 10: 26201.06 + rank 11: 26200.97 + rank 12: 26201.00 + rank 13: 26200.96 + rank 14: 26201.00 + rank 15: 26200.98 + rank 16: 26202.85 + rank 17: 26202.84 + rank 18: 26202.83 + rank 19: 26202.83 + rank 20: 26202.81 + rank 21: 26202.82 + rank 22: 26202.83 + rank 23: 26202.85 + rank 24: 26201.63 + rank 25: 26201.63 + rank 26: 26201.50 + rank 27: 26201.61 + rank 28: 26201.61 + rank 29: 26201.66 + rank 30: 26201.66 + rank 31: 26201.35 + rank 32: 26201.14 + rank 33: 26201.13 + rank 34: 26201.09 + rank 35: 26201.04 + rank 36: 26201.01 + rank 37: 26200.97 + rank 38: 26201.04 + rank 39: 26200.94 + rank 40: 26201.55 + rank 41: 26201.54 + rank 42: 26201.53 + rank 43: 26201.55 + rank 44: 26201.54 + rank 45: 26201.57 + rank 46: 26201.57 + rank 47: 26201.54 + rank 48: 26201.28 + rank 49: 26201.33 + rank 50: 26201.51 + rank 51: 26201.30 + rank 52: 26201.33 + rank 53: 26201.33 + rank 54: 26201.24 + rank 55: 26201.26 + rank 56: 26201.04 + rank 57: 26201.11 + rank 58: 26201.07 + rank 59: 26201.10 + rank 60: 26201.06 + rank 61: 26201.04 + rank 62: 26201.03 + rank 63: 26201.04 + [2026-10-13 16:14:29] iteration 801/ 2384 | consumed samples: 25632 | elapsed time per iteration (ms): 218002.2 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 76958.9 | valid tokens/s: 76958.9 | physical tokens/s: 77968.5 | loss tokens/s: 59954.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.054861E-01 | lm-global-supervised-tokens: 1.307018E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.590296E-02 | lm-token-buffer-mib: 2.346473E+02 | global_load_balancing_loss: 1.001118E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11195.007 | params rms: 0.060617 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:51:37.428075 | finish at 2026-10-17 16:06:06 + [2026-10-13 16:18:13] iteration 802/ 2384 | consumed samples: 25664 | elapsed time per iteration (ms): 224687.0 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.23% | FLOPs estimator: workload-aware | logical tokens/s: 74669.1 | valid tokens/s: 74669.1 | physical tokens/s: 75184.8 | loss tokens/s: 55566.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.667674E-01 | lm-global-supervised-tokens: 1.248516E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.327114E-02 | lm-token-buffer-mib: 2.344921E+02 | global_load_balancing_loss: 9.994614E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11194.970 | params rms: 0.060617 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:44:14.835189 | finish at 2026-10-17 19:02:28 + [2026-10-13 16:21:54] iteration 803/ 2384 | consumed samples: 25696 | elapsed time per iteration (ms): 220991.0 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.39% | FLOPs estimator: workload-aware | logical tokens/s: 75918.1 | valid tokens/s: 75918.0 | physical tokens/s: 76574.9 | loss tokens/s: 56373.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.902967E-01 | lm-global-supervised-tokens: 1.245808E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.941531E-02 | lm-token-buffer-mib: 2.332563E+02 | global_load_balancing_loss: 1.002966E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11194.928 | params rms: 0.060617 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:03:06.737353 | finish at 2026-10-17 17:25:01 + [2026-10-13 16:25:38] iteration 804/ 2384 | consumed samples: 25728 | elapsed time per iteration (ms): 224006.7 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 74895.9 | valid tokens/s: 74895.9 | physical tokens/s: 76001.7 | loss tokens/s: 56572.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.302764E-01 | lm-global-supervised-tokens: 1.267270E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.960136E-02 | lm-token-buffer-mib: 2.359377E+02 | global_load_balancing_loss: 1.001044E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11194.880 | params rms: 0.060617 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:18:50.650101 | finish at 2026-10-17 18:44:29 + [2026-10-13 16:29:19] iteration 805/ 2384 | consumed samples: 25760 | elapsed time per iteration (ms): 220993.2 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 75917.3 | valid tokens/s: 75917.2 | physical tokens/s: 76682.5 | loss tokens/s: 54403.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.112836E-01 | lm-global-supervised-tokens: 1.202273E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.027527E-02 | lm-token-buffer-mib: 2.334200E+02 | global_load_balancing_loss: 9.990526E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11194.839 | params rms: 0.060616 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:55:48.281347 | finish at 2026-10-17 17:25:08 + [2026-10-13 16:33:02] iteration 806/ 2384 | consumed samples: 25792 | elapsed time per iteration (ms): 222668.1 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 75346.2 | valid tokens/s: 75346.2 | physical tokens/s: 76040.1 | loss tokens/s: 58817.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.792860E-01 | lm-global-supervised-tokens: 1.309688E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.130573E-02 | lm-token-buffer-mib: 2.357880E+02 | global_load_balancing_loss: 1.000564E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11194.801 | params rms: 0.060616 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:36:10.213830 | finish at 2026-10-17 18:09:12 + [2026-10-13 16:36:46] iteration 807/ 2384 | consumed samples: 25824 | elapsed time per iteration (ms): 223513.8 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 75061.1 | valid tokens/s: 75061.1 | physical tokens/s: 76135.8 | loss tokens/s: 49382.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.707956E-01 | lm-global-supervised-tokens: 1.103767E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.813771E-02 | lm-token-buffer-mib: 2.348940E+02 | global_load_balancing_loss: 9.993935E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11194.765 | params rms: 0.060616 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:54:41.313962 | finish at 2026-10-17 18:31:27 + [2026-10-13 16:40:29] iteration 808/ 2384 | consumed samples: 25856 | elapsed time per iteration (ms): 223061.7 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.16% | FLOPs estimator: workload-aware | logical tokens/s: 75213.2 | valid tokens/s: 75213.2 | physical tokens/s: 76068.4 | loss tokens/s: 56633.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.954422E-01 | lm-global-supervised-tokens: 1.263281E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.475216E-02 | lm-token-buffer-mib: 2.339704E+02 | global_load_balancing_loss: 1.000435E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11194.726 | params rms: 0.060616 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:39:05.163090 | finish at 2026-10-17 18:19:34 + [2026-10-13 16:44:13] iteration 809/ 2384 | consumed samples: 25888 | elapsed time per iteration (ms): 224613.0 | throughput per GPU (TFLOP/s/GPU): 18.3 | MFU 5.86% | FLOPs estimator: workload-aware | logical tokens/s: 74693.9 | valid tokens/s: 74693.9 | physical tokens/s: 75288.8 | loss tokens/s: 59528.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.244901E-01 | lm-global-supervised-tokens: 1.337079E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.372572E-02 | lm-token-buffer-mib: 2.337787E+02 | global_load_balancing_loss: 1.000844E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11194.686 | params rms: 0.060616 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:16:05.447080 | finish at 2026-10-17 19:00:19 + [2026-10-13 16:48:20] iteration 810/ 2384 | consumed samples: 25920 | elapsed time per iteration (ms): 247219.7 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.25% | FLOPs estimator: workload-aware | logical tokens/s: 67863.5 | valid tokens/s: 67863.5 | physical tokens/s: 68997.2 | loss tokens/s: 55968.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.971841E-01 | lm-global-supervised-tokens: 1.383654E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.676104E-02 | lm-token-buffer-mib: 2.347427E+02 | global_load_balancing_loss: 1.000593E+00 | loss scale: 1.0 | grad norm: 0.424 | params norm: 11194.647 | params rms: 0.060615 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:05:23.842104 | finish at 2026-10-18 04:53:44 + [2026-10-13 16:52:01] iteration 811/ 2384 | consumed samples: 25952 | elapsed time per iteration (ms): 220496.4 | throughput per GPU (TFLOP/s/GPU): 26.3 | MFU 8.42% | FLOPs estimator: workload-aware | logical tokens/s: 76088.0 | valid tokens/s: 76088.0 | physical tokens/s: 76380.7 | loss tokens/s: 56881.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.877990E-01 | lm-global-supervised-tokens: 1.254223E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.115925E-02 | lm-token-buffer-mib: 2.331117E+02 | global_load_balancing_loss: 1.001577E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11194.610 | params rms: 0.060615 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:20:40.902641 | finish at 2026-10-17 17:12:42 + [2026-10-13 16:55:45] iteration 812/ 2384 | consumed samples: 25984 | elapsed time per iteration (ms): 223624.7 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 75023.8 | valid tokens/s: 75023.8 | physical tokens/s: 75579.2 | loss tokens/s: 59207.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.446087E-01 | lm-global-supervised-tokens: 1.324014E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.665939E-02 | lm-token-buffer-mib: 2.341563E+02 | global_load_balancing_loss: 1.000377E+00 | loss scale: 1.0 | grad norm: 0.227 | params norm: 11194.570 | params rms: 0.060615 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:38:58.091849 | finish at 2026-10-17 18:34:43 + [2026-10-13 16:59:32] iteration 813/ 2384 | consumed samples: 26016 | elapsed time per iteration (ms): 227804.8 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 73647.3 | valid tokens/s: 73647.2 | physical tokens/s: 74937.7 | loss tokens/s: 60573.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.105474E-01 | lm-global-supervised-tokens: 1.379897E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.537875E-02 | lm-token-buffer-mib: 2.350952E+02 | global_load_balancing_loss: 9.995657E-01 | loss scale: 1.0 | grad norm: 0.510 | params norm: 11194.528 | params rms: 0.060615 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:24:41.310514 | finish at 2026-10-17 20:24:14 + [2026-10-13 17:03:14] iteration 814/ 2384 | consumed samples: 26048 | elapsed time per iteration (ms): 221679.4 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 75682.3 | valid tokens/s: 75682.3 | physical tokens/s: 76352.4 | loss tokens/s: 57081.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.581076E-01 | lm-global-supervised-tokens: 1.265379E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.868897E-02 | lm-token-buffer-mib: 2.340821E+02 | global_load_balancing_loss: 1.000280E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11194.483 | params rms: 0.060614 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:40:36.619768 | finish at 2026-10-17 17:43:51 + [2026-10-13 17:06:59] iteration 815/ 2384 | consumed samples: 26080 | elapsed time per iteration (ms): 225409.8 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 74429.6 | valid tokens/s: 74429.6 | physical tokens/s: 75063.0 | loss tokens/s: 56539.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.405687E-01 | lm-global-supervised-tokens: 1.274451E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.806890E-02 | lm-token-buffer-mib: 2.344220E+02 | global_load_balancing_loss: 1.000857E+00 | loss scale: 1.0 | grad norm: 0.456 | params norm: 11194.439 | params rms: 0.060614 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:14:27.977031 | finish at 2026-10-17 19:21:27 + [2026-10-13 17:10:42] iteration 816/ 2384 | consumed samples: 26112 | elapsed time per iteration (ms): 222687.8 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.96% | FLOPs estimator: workload-aware | logical tokens/s: 75339.5 | valid tokens/s: 75339.5 | physical tokens/s: 76060.4 | loss tokens/s: 57465.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.108771E-01 | lm-global-supervised-tokens: 1.279679E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.120448E-02 | lm-token-buffer-mib: 2.346417E+02 | global_load_balancing_loss: 9.997471E-01 | loss scale: 1.0 | grad norm: 0.270 | params norm: 11194.394 | params rms: 0.060614 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:59:34.495338 | finish at 2026-10-17 18:10:17 + [2026-10-13 17:14:23] iteration 817/ 2384 | consumed samples: 26144 | elapsed time per iteration (ms): 220992.1 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 75917.6 | valid tokens/s: 75917.6 | physical tokens/s: 76326.4 | loss tokens/s: 55463.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.400319E-01 | lm-global-supervised-tokens: 1.225700E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.518274E-02 | lm-token-buffer-mib: 2.339206E+02 | global_load_balancing_loss: 1.000627E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11194.350 | params rms: 0.060614 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:11:34.597537 | finish at 2026-10-17 17:25:58 + [2026-10-13 17:18:05] iteration 818/ 2384 | consumed samples: 26176 | elapsed time per iteration (ms): 221639.3 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 75696.0 | valid tokens/s: 75696.0 | physical tokens/s: 76258.5 | loss tokens/s: 54958.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.235794E-01 | lm-global-supervised-tokens: 1.218106E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.138640E-02 | lm-token-buffer-mib: 2.330417E+02 | global_load_balancing_loss: 1.000439E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11194.306 | params rms: 0.060614 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:24:47.087965 | finish at 2026-10-17 17:42:52 + [2026-10-13 17:21:52] iteration 819/ 2384 | consumed samples: 26208 | elapsed time per iteration (ms): 227179.9 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 73849.8 | valid tokens/s: 73849.8 | physical tokens/s: 74992.0 | loss tokens/s: 51526.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.181250E-01 | lm-global-supervised-tokens: 1.170580E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.840033E-02 | lm-token-buffer-mib: 2.359761E+02 | global_load_balancing_loss: 1.001973E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11194.263 | params rms: 0.060613 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:45:36.548616 | finish at 2026-10-17 20:07:29 + [2026-10-13 17:25:56] iteration 820/ 2384 | consumed samples: 26240 | elapsed time per iteration (ms): 244461.4 | throughput per GPU (TFLOP/s/GPU): 17.8 | MFU 5.70% | FLOPs estimator: workload-aware | logical tokens/s: 68629.3 | valid tokens/s: 68629.3 | physical tokens/s: 69796.1 | loss tokens/s: 57063.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.399197E-01 | lm-global-supervised-tokens: 1.394975E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.356522E-02 | lm-token-buffer-mib: 2.353237E+02 | global_load_balancing_loss: 9.996774E-01 | loss scale: 1.0 | grad norm: 0.202 | params norm: 11194.219 | params rms: 0.060613 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:12:17.641955 | finish at 2026-10-18 03:38:14 + [2026-10-13 17:29:36] iteration 821/ 2384 | consumed samples: 26272 | elapsed time per iteration (ms): 219235.5 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 76525.9 | valid tokens/s: 76525.9 | physical tokens/s: 77563.8 | loss tokens/s: 50028.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.844905E-01 | lm-global-supervised-tokens: 1.096806E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.135324E-02 | lm-token-buffer-mib: 2.332923E+02 | global_load_balancing_loss: 1.000015E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11194.174 | params rms: 0.060613 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:11:05.132115 | finish at 2026-10-17 16:40:41 + [2026-10-13 17:33:20] iteration 822/ 2384 | consumed samples: 26304 | elapsed time per iteration (ms): 224746.8 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 74649.2 | valid tokens/s: 74649.2 | physical tokens/s: 75132.1 | loss tokens/s: 57109.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.354346E-01 | lm-global-supervised-tokens: 1.283519E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.546447E-02 | lm-token-buffer-mib: 2.340027E+02 | global_load_balancing_loss: 9.991523E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11194.131 | params rms: 0.060613 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:30:54.564080 | finish at 2026-10-17 19:04:15 + [2026-10-13 17:37:09] iteration 823/ 2384 | consumed samples: 26336 | elapsed time per iteration (ms): 228908.0 | throughput per GPU (TFLOP/s/GPU): 18.0 | MFU 5.78% | FLOPs estimator: workload-aware | logical tokens/s: 73292.4 | valid tokens/s: 73292.4 | physical tokens/s: 75222.4 | loss tokens/s: 55378.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.502961E-01 | lm-global-supervised-tokens: 1.267646E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.505246E-02 | lm-token-buffer-mib: 2.382351E+02 | global_load_balancing_loss: 9.995543E-01 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11194.086 | params rms: 0.060612 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:15:25.379056 | finish at 2026-10-17 20:52:35 + [2026-10-13 17:40:50] iteration 824/ 2384 | consumed samples: 26368 | elapsed time per iteration (ms): 220380.0 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 76128.6 | valid tokens/s: 76128.6 | physical tokens/s: 77220.8 | loss tokens/s: 57641.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.749098E-01 | lm-global-supervised-tokens: 1.270306E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.697768E-02 | lm-token-buffer-mib: 2.348090E+02 | global_load_balancing_loss: 1.000511E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11194.041 | params rms: 0.060612 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:29:52.861176 | finish at 2026-10-17 17:10:43 + [2026-10-13 17:44:33] iteration 825/ 2384 | consumed samples: 26400 | elapsed time per iteration (ms): 223598.1 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 75032.9 | valid tokens/s: 75032.9 | physical tokens/s: 75952.0 | loss tokens/s: 55462.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.634325E-01 | lm-global-supervised-tokens: 1.240121E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.140697E-02 | lm-token-buffer-mib: 2.348587E+02 | global_load_balancing_loss: 1.001702E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11193.993 | params rms: 0.060612 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:49:49.377603 | finish at 2026-10-17 18:34:23 + [2026-10-13 17:48:19] iteration 826/ 2384 | consumed samples: 26432 | elapsed time per iteration (ms): 226170.7 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.92% | FLOPs estimator: workload-aware | logical tokens/s: 74179.2 | valid tokens/s: 74179.2 | physical tokens/s: 74844.7 | loss tokens/s: 61750.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.377500E-01 | lm-global-supervised-tokens: 1.396619E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.083885E-02 | lm-token-buffer-mib: 2.350591E+02 | global_load_balancing_loss: 1.002187E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11193.945 | params rms: 0.060612 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:52:53.999746 | finish at 2026-10-17 19:41:13 + [2026-10-13 17:52:05] iteration 827/ 2384 | consumed samples: 26464 | elapsed time per iteration (ms): 225256.5 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 74480.2 | valid tokens/s: 74480.2 | physical tokens/s: 74931.7 | loss tokens/s: 54676.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.652484E-01 | lm-global-supervised-tokens: 1.231633E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.221072E-02 | lm-token-buffer-mib: 2.331365E+02 | global_load_balancing_loss: 1.001146E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11193.896 | params rms: 0.060611 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:25:24.413199 | finish at 2026-10-17 19:17:29 + [2026-10-13 17:55:47] iteration 828/ 2384 | consumed samples: 26496 | elapsed time per iteration (ms): 222216.4 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.42% | FLOPs estimator: workload-aware | logical tokens/s: 75499.2 | valid tokens/s: 75499.2 | physical tokens/s: 76065.1 | loss tokens/s: 55322.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.254675E-01 | lm-global-supervised-tokens: 1.229364E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.368602E-02 | lm-token-buffer-mib: 2.331855E+02 | global_load_balancing_loss: 1.001053E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11193.846 | params rms: 0.060611 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:02:48.746822 | finish at 2026-10-17 17:58:36 + [2026-10-13 17:59:30] iteration 829/ 2384 | consumed samples: 26528 | elapsed time per iteration (ms): 222700.1 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 75335.4 | valid tokens/s: 75335.4 | physical tokens/s: 75480.3 | loss tokens/s: 56709.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.647353E-01 | lm-global-supervised-tokens: 1.262922E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.224970E-02 | lm-token-buffer-mib: 2.323230E+02 | global_load_balancing_loss: 1.000517E+00 | loss scale: 1.0 | grad norm: 0.180 | params norm: 11193.799 | params rms: 0.060611 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:11:38.599062 | finish at 2026-10-17 18:11:08 + [2026-10-13 18:03:33] iteration 830/ 2384 | consumed samples: 26560 | elapsed time per iteration (ms): 243271.8 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 68964.7 | valid tokens/s: 68964.7 | physical tokens/s: 69797.0 | loss tokens/s: 52539.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.406897E-01 | lm-global-supervised-tokens: 1.278137E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.367497E-02 | lm-token-buffer-mib: 2.333785E+02 | global_load_balancing_loss: 1.001721E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11193.754 | params rms: 0.060611 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:00:44.337250 | finish at 2026-10-18 03:04:17 + [2026-10-13 18:07:16] iteration 831/ 2384 | consumed samples: 26592 | elapsed time per iteration (ms): 223019.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75227.5 | valid tokens/s: 75227.5 | physical tokens/s: 76020.5 | loss tokens/s: 60073.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.791529E-01 | lm-global-supervised-tokens: 1.339744E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.507560E-02 | lm-token-buffer-mib: 2.341376E+02 | global_load_balancing_loss: 1.001026E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11193.709 | params rms: 0.060610 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:12:29.273159 | finish at 2026-10-17 18:19:45 + [2026-10-13 18:10:59] iteration 832/ 2384 | consumed samples: 26624 | elapsed time per iteration (ms): 223109.2 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 75197.3 | valid tokens/s: 75197.3 | physical tokens/s: 76054.2 | loss tokens/s: 58664.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.678255E-01 | lm-global-supervised-tokens: 1.308849E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.541106E-02 | lm-token-buffer-mib: 2.357405E+02 | global_load_balancing_loss: 9.992364E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11193.667 | params rms: 0.060610 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:11:05.479881 | finish at 2026-10-17 18:22:05 + [2026-10-13 18:14:41] iteration 833/ 2384 | consumed samples: 26656 | elapsed time per iteration (ms): 221611.3 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.21% | FLOPs estimator: workload-aware | logical tokens/s: 75705.5 | valid tokens/s: 75705.5 | physical tokens/s: 76367.2 | loss tokens/s: 56278.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.766150E-01 | lm-global-supervised-tokens: 1.247200E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.164368E-02 | lm-token-buffer-mib: 2.349435E+02 | global_load_balancing_loss: 9.986175E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11193.629 | params rms: 0.060610 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:28:39.068230 | finish at 2026-10-17 17:43:20 + [2026-10-13 18:18:25] iteration 834/ 2384 | consumed samples: 26688 | elapsed time per iteration (ms): 224328.5 | throughput per GPU (TFLOP/s/GPU): 25.3 | MFU 8.11% | FLOPs estimator: workload-aware | logical tokens/s: 74788.5 | valid tokens/s: 74788.5 | physical tokens/s: 75299.9 | loss tokens/s: 54673.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.447333E-01 | lm-global-supervised-tokens: 1.226471E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.790277E-02 | lm-token-buffer-mib: 2.336169E+02 | global_load_balancing_loss: 9.994605E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11193.593 | params rms: 0.060610 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:35:09.161377 | finish at 2026-10-17 18:53:34 + [2026-10-13 18:22:10] iteration 835/ 2384 | consumed samples: 26720 | elapsed time per iteration (ms): 224660.0 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.95% | FLOPs estimator: workload-aware | logical tokens/s: 74678.2 | valid tokens/s: 74678.2 | physical tokens/s: 75424.1 | loss tokens/s: 59543.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.359368E-01 | lm-global-supervised-tokens: 1.337693E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.335631E-02 | lm-token-buffer-mib: 2.350654E+02 | global_load_balancing_loss: 9.985675E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11193.557 | params rms: 0.060609 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:39:58.272180 | finish at 2026-10-17 19:02:08 + [2026-10-13 18:25:51] iteration 836/ 2384 | consumed samples: 26752 | elapsed time per iteration (ms): 220942.8 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 5.99% | FLOPs estimator: workload-aware | logical tokens/s: 75934.6 | valid tokens/s: 75934.5 | physical tokens/s: 76814.7 | loss tokens/s: 60000.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.811486E-01 | lm-global-supervised-tokens: 1.325673E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.410581E-02 | lm-token-buffer-mib: 2.344851E+02 | global_load_balancing_loss: 1.002022E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11193.519 | params rms: 0.060609 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:00:19.512783 | finish at 2026-10-17 17:26:10 + [2026-10-13 18:29:34] iteration 837/ 2384 | consumed samples: 26784 | elapsed time per iteration (ms): 223560.9 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 75045.2 | valid tokens/s: 75045.2 | physical tokens/s: 75607.9 | loss tokens/s: 54597.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.741410E-01 | lm-global-supervised-tokens: 1.220595E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.672908E-02 | lm-token-buffer-mib: 2.330737E+02 | global_load_balancing_loss: 1.001037E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11193.478 | params rms: 0.060609 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:04:08.663203 | finish at 2026-10-17 18:33:43 + [2026-10-13 18:33:15] iteration 838/ 2384 | consumed samples: 26816 | elapsed time per iteration (ms): 220949.4 | throughput per GPU (TFLOP/s/GPU): 16.9 | MFU 5.43% | FLOPs estimator: workload-aware | logical tokens/s: 75932.3 | valid tokens/s: 75932.3 | physical tokens/s: 76687.3 | loss tokens/s: 58874.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.096476E-01 | lm-global-supervised-tokens: 1.300825E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.311281E-02 | lm-token-buffer-mib: 2.340762E+02 | global_load_balancing_loss: 1.001466E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11193.438 | params rms: 0.060609 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:53:07.733986 | finish at 2026-10-17 17:26:23 + [2026-10-13 18:36:58] iteration 839/ 2384 | consumed samples: 26848 | elapsed time per iteration (ms): 222445.1 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 75421.7 | valid tokens/s: 75421.7 | physical tokens/s: 75897.4 | loss tokens/s: 58465.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.124433E-01 | lm-global-supervised-tokens: 1.300545E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.257164E-02 | lm-token-buffer-mib: 2.330382E+02 | global_load_balancing_loss: 1.000996E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11193.399 | params rms: 0.060609 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:27:57.668924 | finish at 2026-10-17 18:04:55 + [2026-10-13 18:41:08] iteration 840/ 2384 | consumed samples: 26880 | elapsed time per iteration (ms): 250615.5 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 66944.0 | valid tokens/s: 66944.0 | physical tokens/s: 67617.2 | loss tokens/s: 51401.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.876557E-01 | lm-global-supervised-tokens: 1.288208E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.420423E-02 | lm-token-buffer-mib: 2.364014E+02 | global_load_balancing_loss: 1.002533E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11193.359 | params rms: 0.060608 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:29:10.366930 | finish at 2026-10-18 06:10:19 +times across ranks (ms): + save-checkpoint: + rank 0: 24829.54 + rank 1: 24829.53 + rank 2: 24829.54 + rank 3: 24829.54 + rank 4: 24829.53 + rank 5: 24829.54 + rank 6: 24829.55 + rank 7: 24829.51 + rank 8: 24830.38 + rank 9: 24830.42 + rank 10: 24830.43 + rank 11: 24830.38 + rank 12: 24830.41 + rank 13: 24830.38 + rank 14: 24830.39 + rank 15: 24830.38 + rank 16: 24830.71 + rank 17: 24830.69 + rank 18: 24830.71 + rank 19: 24830.74 + rank 20: 24830.68 + rank 21: 24830.67 + rank 22: 24830.70 + rank 23: 24830.73 + rank 24: 24830.39 + rank 25: 24830.40 + rank 26: 24830.36 + rank 27: 24830.29 + rank 28: 24830.33 + rank 29: 24830.32 + rank 30: 24830.28 + rank 31: 24830.41 + rank 32: 24830.64 + rank 33: 24830.84 + rank 34: 24830.68 + rank 35: 24830.66 + rank 36: 24830.74 + rank 37: 24830.65 + rank 38: 24830.64 + rank 39: 24830.75 + rank 40: 24830.55 + rank 41: 24830.60 + rank 42: 24830.54 + rank 43: 24830.56 + rank 44: 24830.59 + rank 45: 24830.57 + rank 46: 24830.54 + rank 47: 24830.54 + rank 48: 24830.26 + rank 49: 24830.20 + rank 50: 24830.35 + rank 51: 24830.29 + rank 52: 24830.27 + rank 53: 24830.14 + rank 54: 24830.14 + rank 55: 24830.08 + rank 56: 24830.44 + rank 57: 24830.46 + rank 58: 24830.46 + rank 59: 24830.30 + rank 60: 24830.47 + rank 61: 24830.45 + rank 62: 24830.43 + rank 63: 24830.44 + [2026-10-13 18:45:29] iteration 841/ 2384 | consumed samples: 26912 | elapsed time per iteration (ms): 235636.7 | throughput per GPU (TFLOP/s/GPU): 26.1 | MFU 8.37% | FLOPs estimator: workload-aware | logical tokens/s: 71199.3 | valid tokens/s: 71199.3 | physical tokens/s: 71913.5 | loss tokens/s: 56249.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.226133E-01 | lm-global-supervised-tokens: 1.325433E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.493320E-01 | lm-token-buffer-mib: 2.330402E+02 | global_load_balancing_loss: 1.002613E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11193.319 | params rms: 0.060608 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:59:47.485726 | finish at 2026-10-17 23:45:16 + failed to log checkpoint /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0000840 as wandb artifact: Unable to connect to server to verify API token. + [2026-10-13 18:49:20] iteration 842/ 2384 | consumed samples: 26944 | elapsed time per iteration (ms): 231442.8 | throughput per GPU (TFLOP/s/GPU): 16.9 | MFU 5.43% | FLOPs estimator: workload-aware | logical tokens/s: 72489.6 | valid tokens/s: 72489.6 | physical tokens/s: 73466.6 | loss tokens/s: 56944.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.417447E-01 | lm-global-supervised-tokens: 1.317940E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.223036E-02 | lm-token-buffer-mib: 2.338121E+02 | global_load_balancing_loss: 9.993495E-01 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11193.277 | params rms: 0.060608 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:08:04.842889 | finish at 2026-10-17 21:57:25 + [2026-10-13 18:53:03] iteration 843/ 2384 | consumed samples: 26976 | elapsed time per iteration (ms): 222586.7 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75373.7 | valid tokens/s: 75373.7 | physical tokens/s: 76875.3 | loss tokens/s: 59864.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.294487E-01 | lm-global-supervised-tokens: 1.332503E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.366638E-02 | lm-token-buffer-mib: 2.350647E+02 | global_load_balancing_loss: 9.992112E-01 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11193.234 | params rms: 0.060608 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:16:46.067902 | finish at 2026-10-17 18:09:49 + [2026-10-13 18:56:44] iteration 844/ 2384 | consumed samples: 27008 | elapsed time per iteration (ms): 221288.3 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75816.0 | valid tokens/s: 75816.0 | physical tokens/s: 76571.0 | loss tokens/s: 58199.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.916058E-01 | lm-global-supervised-tokens: 1.287877E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.327050E-02 | lm-token-buffer-mib: 2.337727E+02 | global_load_balancing_loss: 9.983931E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11193.193 | params rms: 0.060607 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:39:43.967738 | finish at 2026-10-17 17:36:28 + [2026-10-13 19:00:25] iteration 845/ 2384 | consumed samples: 27040 | elapsed time per iteration (ms): 221424.2 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 75769.5 | valid tokens/s: 75769.5 | physical tokens/s: 76370.5 | loss tokens/s: 59470.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.973121E-01 | lm-global-supervised-tokens: 1.316819E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.515691E-02 | lm-token-buffer-mib: 2.328472E+02 | global_load_balancing_loss: 9.985907E-01 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11193.152 | params rms: 0.060607 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:39:31.845357 | finish at 2026-10-17 17:39:57 + [2026-10-13 19:04:07] iteration 846/ 2384 | consumed samples: 27072 | elapsed time per iteration (ms): 221268.0 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 75823.0 | valid tokens/s: 75823.0 | physical tokens/s: 76064.0 | loss tokens/s: 57700.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.696271E-01 | lm-global-supervised-tokens: 1.276731E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.256251E-02 | lm-token-buffer-mib: 2.327261E+02 | global_load_balancing_loss: 1.000217E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11193.110 | params rms: 0.060607 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:31:50.213162 | finish at 2026-10-17 17:35:57 + [2026-10-13 19:07:52] iteration 847/ 2384 | consumed samples: 27104 | elapsed time per iteration (ms): 225012.6 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 74561.1 | valid tokens/s: 74561.1 | physical tokens/s: 76079.2 | loss tokens/s: 60430.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.804529E-01 | lm-global-supervised-tokens: 1.359768E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.593379E-02 | lm-token-buffer-mib: 2.395210E+02 | global_load_balancing_loss: 1.000044E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11193.068 | params rms: 0.060607 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:04:04.298322 | finish at 2026-10-17 19:11:56 + [2026-10-13 19:11:31] iteration 848/ 2384 | consumed samples: 27136 | elapsed time per iteration (ms): 219212.6 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 76533.9 | valid tokens/s: 76533.9 | physical tokens/s: 77347.1 | loss tokens/s: 58864.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.976988E-01 | lm-global-supervised-tokens: 1.290380E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.826616E-02 | lm-token-buffer-mib: 2.330449E+02 | global_load_balancing_loss: 1.001069E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11193.025 | params rms: 0.060607 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:31:50.595337 | finish at 2026-10-17 16:43:22 + [2026-10-13 19:15:13] iteration 849/ 2384 | consumed samples: 27168 | elapsed time per iteration (ms): 221652.4 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 75691.5 | valid tokens/s: 75691.5 | physical tokens/s: 76578.3 | loss tokens/s: 59820.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.330699E-01 | lm-global-supervised-tokens: 1.325930E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.870280E-02 | lm-token-buffer-mib: 2.328116E+02 | global_load_balancing_loss: 1.002499E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11192.980 | params rms: 0.060606 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:30:36.425608 | finish at 2026-10-17 17:45:49 + [2026-10-13 19:19:16] iteration 850/ 2384 | consumed samples: 27200 | elapsed time per iteration (ms): 243935.6 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 68777.2 | valid tokens/s: 68777.2 | physical tokens/s: 69769.5 | loss tokens/s: 53680.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.337282E-01 | lm-global-supervised-tokens: 1.309462E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.187716E-02 | lm-token-buffer-mib: 2.336841E+02 | global_load_balancing_loss: 1.001484E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11192.933 | params rms: 0.060606 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:56:37.180475 | finish at 2026-10-18 03:15:54 + [2026-10-13 19:23:04] iteration 851/ 2384 | consumed samples: 27232 | elapsed time per iteration (ms): 227377.2 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 73785.7 | valid tokens/s: 73785.7 | physical tokens/s: 74506.4 | loss tokens/s: 57450.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.666534E-01 | lm-global-supervised-tokens: 1.306283E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.444799E-02 | lm-token-buffer-mib: 2.344109E+02 | global_load_balancing_loss: 1.001802E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11192.882 | params rms: 0.060606 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:49:29.308101 | finish at 2026-10-17 20:12:33 + [2026-10-13 19:26:47] iteration 852/ 2384 | consumed samples: 27264 | elapsed time per iteration (ms): 222932.2 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75257.0 | valid tokens/s: 75257.0 | physical tokens/s: 76080.1 | loss tokens/s: 58968.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.212005E-01 | lm-global-supervised-tokens: 1.314603E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.388327E-02 | lm-token-buffer-mib: 2.345519E+02 | global_load_balancing_loss: 1.000620E+00 | loss scale: 1.0 | grad norm: 0.274 | params norm: 11192.832 | params rms: 0.060606 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:52:12.114990 | finish at 2026-10-17 18:18:59 + [2026-10-13 19:30:33] iteration 853/ 2384 | consumed samples: 27296 | elapsed time per iteration (ms): 225844.9 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 74286.4 | valid tokens/s: 74286.4 | physical tokens/s: 74798.8 | loss tokens/s: 53909.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.575367E-01 | lm-global-supervised-tokens: 1.217510E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.264137E-02 | lm-token-buffer-mib: 2.338782E+02 | global_load_balancing_loss: 1.000573E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11192.782 | params rms: 0.060605 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:02:48.508884 | finish at 2026-10-17 19:33:21 + [2026-10-13 19:34:16] iteration 854/ 2384 | consumed samples: 27328 | elapsed time per iteration (ms): 222958.2 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.36% | FLOPs estimator: workload-aware | logical tokens/s: 75248.2 | valid tokens/s: 75248.2 | physical tokens/s: 75787.3 | loss tokens/s: 59276.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.002176E-01 | lm-global-supervised-tokens: 1.321618E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.636110E-02 | lm-token-buffer-mib: 2.331548E+02 | global_load_balancing_loss: 9.999468E-01 | loss scale: 1.0 | grad norm: 0.289 | params norm: 11192.735 | params rms: 0.060605 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:45:25.998180 | finish at 2026-10-17 18:19:42 + [2026-10-13 19:37:55] iteration 855/ 2384 | consumed samples: 27360 | elapsed time per iteration (ms): 219418.5 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 76462.1 | valid tokens/s: 76462.1 | physical tokens/s: 77054.3 | loss tokens/s: 59781.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.012111E-01 | lm-global-supervised-tokens: 1.311726E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.627972E-02 | lm-token-buffer-mib: 2.347130E+02 | global_load_balancing_loss: 9.991597E-01 | loss scale: 1.0 | grad norm: 0.277 | params norm: 11192.687 | params rms: 0.060605 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:11:30.848506 | finish at 2026-10-17 16:49:26 + [2026-10-13 19:41:39] iteration 856/ 2384 | consumed samples: 27392 | elapsed time per iteration (ms): 224051.1 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 74881.2 | valid tokens/s: 74881.2 | physical tokens/s: 75413.1 | loss tokens/s: 58093.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.553335E-01 | lm-global-supervised-tokens: 1.301589E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.166077E-02 | lm-token-buffer-mib: 2.342206E+02 | global_load_balancing_loss: 1.000132E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11192.639 | params rms: 0.060605 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:05:50.041115 | finish at 2026-10-17 18:47:29 + [2026-10-13 19:45:21] iteration 857/ 2384 | consumed samples: 27424 | elapsed time per iteration (ms): 222349.4 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75454.2 | valid tokens/s: 75454.2 | physical tokens/s: 75684.0 | loss tokens/s: 55498.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.852357E-01 | lm-global-supervised-tokens: 1.233994E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.600938E-02 | lm-token-buffer-mib: 2.327923E+02 | global_load_balancing_loss: 1.000494E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11192.592 | params rms: 0.060604 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:18:47.491271 | finish at 2026-10-17 18:04:09 + [2026-10-13 19:49:01] iteration 858/ 2384 | consumed samples: 27456 | elapsed time per iteration (ms): 219614.1 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 76394.0 | valid tokens/s: 76394.0 | physical tokens/s: 76944.9 | loss tokens/s: 58813.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.464020E-01 | lm-global-supervised-tokens: 1.291634E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.908604E-02 | lm-token-buffer-mib: 2.333309E+02 | global_load_balancing_loss: 1.000573E+00 | loss scale: 1.0 | grad norm: 0.206 | params norm: 11192.549 | params rms: 0.060604 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:05:31.145675 | finish at 2026-10-17 16:54:32 + [2026-10-13 19:52:43] iteration 859/ 2384 | consumed samples: 27488 | elapsed time per iteration (ms): 222285.6 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 75475.9 | valid tokens/s: 75475.9 | physical tokens/s: 76245.2 | loss tokens/s: 58001.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.442196E-01 | lm-global-supervised-tokens: 1.289298E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.299152E-02 | lm-token-buffer-mib: 2.350679E+02 | global_load_balancing_loss: 1.000430E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11192.512 | params rms: 0.060604 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:09:45.611182 | finish at 2026-10-17 18:02:29 + [2026-10-13 19:56:46] iteration 860/ 2384 | consumed samples: 27520 | elapsed time per iteration (ms): 243148.2 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 68999.8 | valid tokens/s: 68999.8 | physical tokens/s: 69566.1 | loss tokens/s: 52627.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.373590E-01 | lm-global-supervised-tokens: 1.279634E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.555645E-02 | lm-token-buffer-mib: 2.332253E+02 | global_load_balancing_loss: 9.998353E-01 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11192.477 | params rms: 0.060604 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:55:57.925527 | finish at 2026-10-18 02:52:44 + [2026-10-13 20:00:27] iteration 861/ 2384 | consumed samples: 27552 | elapsed time per iteration (ms): 220684.4 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 76023.5 | valid tokens/s: 76023.5 | physical tokens/s: 77191.7 | loss tokens/s: 57885.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.887533E-01 | lm-global-supervised-tokens: 1.277434E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.777363E-02 | lm-token-buffer-mib: 2.339445E+02 | global_load_balancing_loss: 1.001099E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11192.434 | params rms: 0.060603 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:21:42.306466 | finish at 2026-10-17 17:22:09 + [2026-10-13 20:04:07] iteration 862/ 2384 | consumed samples: 27584 | elapsed time per iteration (ms): 219834.0 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 76317.5 | valid tokens/s: 76317.5 | physical tokens/s: 77081.0 | loss tokens/s: 56721.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.422428E-01 | lm-global-supervised-tokens: 1.246939E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.683330E-02 | lm-token-buffer-mib: 2.335298E+02 | global_load_balancing_loss: 1.000329E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11192.390 | params rms: 0.060603 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:56:27.303535 | finish at 2026-10-17 17:00:34 + [2026-10-13 20:07:48] iteration 863/ 2384 | consumed samples: 27616 | elapsed time per iteration (ms): 221236.1 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75833.8 | valid tokens/s: 75833.8 | physical tokens/s: 76482.9 | loss tokens/s: 59192.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.451305E-01 | lm-global-supervised-tokens: 1.309553E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.327098E-02 | lm-token-buffer-mib: 2.332395E+02 | global_load_balancing_loss: 9.998851E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11192.349 | params rms: 0.060603 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:28:20.081927 | finish at 2026-10-17 17:36:08 + [2026-10-13 20:11:29] iteration 864/ 2384 | consumed samples: 27648 | elapsed time per iteration (ms): 221170.9 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.25% | FLOPs estimator: workload-aware | logical tokens/s: 75856.3 | valid tokens/s: 75856.3 | physical tokens/s: 76418.6 | loss tokens/s: 60468.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.513175E-01 | lm-global-supervised-tokens: 1.337388E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.623905E-02 | lm-token-buffer-mib: 2.336843E+02 | global_load_balancing_loss: 1.001273E+00 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11192.308 | params rms: 0.060603 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:22:59.704380 | finish at 2026-10-17 17:34:29 + [2026-10-13 20:15:12] iteration 865/ 2384 | consumed samples: 27680 | elapsed time per iteration (ms): 222591.4 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 75372.1 | valid tokens/s: 75372.1 | physical tokens/s: 76205.8 | loss tokens/s: 61224.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.825656E-01 | lm-global-supervised-tokens: 1.362803E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.796472E-02 | lm-token-buffer-mib: 2.336288E+02 | global_load_balancing_loss: 1.001817E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11192.266 | params rms: 0.060602 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:55:16.377384 | finish at 2026-10-17 18:10:28 + [2026-10-13 20:18:57] iteration 866/ 2384 | consumed samples: 27712 | elapsed time per iteration (ms): 225105.6 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 74530.4 | valid tokens/s: 74530.4 | physical tokens/s: 76055.8 | loss tokens/s: 57915.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.557159E-01 | lm-global-supervised-tokens: 1.303700E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.182775E-02 | lm-token-buffer-mib: 2.368633E+02 | global_load_balancing_loss: 1.000969E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11192.220 | params rms: 0.060602 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:55:10.268407 | finish at 2026-10-17 19:14:07 + [2026-10-13 20:22:43] iteration 867/ 2384 | consumed samples: 27744 | elapsed time per iteration (ms): 226293.3 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 74139.2 | valid tokens/s: 74139.2 | physical tokens/s: 74640.1 | loss tokens/s: 57321.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.880495E-01 | lm-global-supervised-tokens: 1.297157E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.787601E-02 | lm-token-buffer-mib: 2.326698E+02 | global_load_balancing_loss: 1.001303E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11192.175 | params rms: 0.060602 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:21:26.955138 | finish at 2026-10-17 19:44:10 + [2026-10-13 20:26:28] iteration 868/ 2384 | consumed samples: 27776 | elapsed time per iteration (ms): 224318.4 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 74791.9 | valid tokens/s: 74791.9 | physical tokens/s: 75644.2 | loss tokens/s: 58187.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.715529E-01 | lm-global-supervised-tokens: 1.305252E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.247917E-02 | lm-token-buffer-mib: 2.346296E+02 | global_load_balancing_loss: 1.000471E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11192.132 | params rms: 0.060602 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:27:46.721727 | finish at 2026-10-17 18:54:14 + [2026-10-13 20:30:08] iteration 869/ 2384 | consumed samples: 27808 | elapsed time per iteration (ms): 219849.6 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.16% | FLOPs estimator: workload-aware | logical tokens/s: 76312.1 | valid tokens/s: 76312.1 | physical tokens/s: 76946.3 | loss tokens/s: 62602.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.311321E-01 | lm-global-supervised-tokens: 1.376320E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.262559E-02 | lm-token-buffer-mib: 2.336524E+02 | global_load_balancing_loss: 9.987952E-01 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11192.091 | params rms: 0.060602 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:31:12.171929 | finish at 2026-10-17 17:01:20 + [2026-10-13 20:34:11] iteration 870/ 2384 | consumed samples: 27840 | elapsed time per iteration (ms): 243120.4 | throughput per GPU (TFLOP/s/GPU): 18.0 | MFU 5.77% | FLOPs estimator: workload-aware | logical tokens/s: 69007.8 | valid tokens/s: 69007.8 | physical tokens/s: 69546.2 | loss tokens/s: 53002.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.854300E-01 | lm-global-supervised-tokens: 1.288589E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.786050E-02 | lm-token-buffer-mib: 2.335450E+02 | global_load_balancing_loss: 9.992610E-01 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11192.052 | params rms: 0.060601 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:14:44.211168 | finish at 2026-10-18 02:48:55 + [2026-10-13 20:37:53] iteration 871/ 2384 | consumed samples: 27872 | elapsed time per iteration (ms): 222716.7 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.52% | FLOPs estimator: workload-aware | logical tokens/s: 75329.7 | valid tokens/s: 75329.7 | physical tokens/s: 75787.9 | loss tokens/s: 51572.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.635157E-01 | lm-global-supervised-tokens: 1.148602E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.735165E-02 | lm-token-buffer-mib: 2.339758E+02 | global_load_balancing_loss: 1.000707E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11192.011 | params rms: 0.060601 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:36:10.381646 | finish at 2026-10-17 18:14:04 + [2026-10-13 20:41:38] iteration 872/ 2384 | consumed samples: 27904 | elapsed time per iteration (ms): 225040.7 | throughput per GPU (TFLOP/s/GPU): 17.2 | MFU 5.52% | FLOPs estimator: workload-aware | logical tokens/s: 74551.8 | valid tokens/s: 74551.8 | physical tokens/s: 76348.4 | loss tokens/s: 55022.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.919465E-01 | lm-global-supervised-tokens: 1.238228E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.630390E-02 | lm-token-buffer-mib: 2.354722E+02 | global_load_balancing_loss: 9.994236E-01 | loss scale: 1.0 | grad norm: 0.391 | params norm: 11191.967 | params rms: 0.060601 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:31:01.593853 | finish at 2026-10-17 19:12:40 + [2026-10-13 20:45:18] iteration 873/ 2384 | consumed samples: 27936 | elapsed time per iteration (ms): 219566.7 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 76410.5 | valid tokens/s: 76410.5 | physical tokens/s: 77097.0 | loss tokens/s: 58085.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.400709E-01 | lm-global-supervised-tokens: 1.275370E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.483779E-02 | lm-token-buffer-mib: 2.337541E+02 | global_load_balancing_loss: 1.000025E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11191.923 | params rms: 0.060601 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:09:25.261697 | finish at 2026-10-17 16:54:43 + [2026-10-13 20:49:03] iteration 874/ 2384 | consumed samples: 27968 | elapsed time per iteration (ms): 225002.8 | throughput per GPU (TFLOP/s/GPU): 26.0 | MFU 8.34% | FLOPs estimator: workload-aware | logical tokens/s: 74564.4 | valid tokens/s: 74564.4 | physical tokens/s: 75036.9 | loss tokens/s: 53006.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.500792E-01 | lm-global-supervised-tokens: 1.192669E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.467960E-02 | lm-token-buffer-mib: 2.333640E+02 | global_load_balancing_loss: 1.000321E+00 | loss scale: 1.0 | grad norm: 0.390 | params norm: 11191.879 | params rms: 0.060600 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:22:34.280183 | finish at 2026-10-17 19:11:37 + [2026-10-13 20:52:47] iteration 875/ 2384 | consumed samples: 28000 | elapsed time per iteration (ms): 224328.7 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.29% | FLOPs estimator: workload-aware | logical tokens/s: 74788.3 | valid tokens/s: 74788.3 | physical tokens/s: 75150.0 | loss tokens/s: 55258.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.946882E-01 | lm-global-supervised-tokens: 1.239610E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.597067E-02 | lm-token-buffer-mib: 2.322022E+02 | global_load_balancing_loss: 1.001311E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11191.826 | params rms: 0.060600 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:01:52.053730 | finish at 2026-10-17 18:54:39 + [2026-10-13 20:56:28] iteration 876/ 2384 | consumed samples: 28032 | elapsed time per iteration (ms): 221036.2 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 75902.6 | valid tokens/s: 75902.6 | physical tokens/s: 76541.9 | loss tokens/s: 58969.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.567458E-01 | lm-global-supervised-tokens: 1.303448E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.457183E-02 | lm-token-buffer-mib: 2.343786E+02 | global_load_balancing_loss: 1.000939E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11191.780 | params rms: 0.060600 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:35:22.624905 | finish at 2026-10-17 17:31:51 + [2026-10-13 21:00:08] iteration 877/ 2384 | consumed samples: 28064 | elapsed time per iteration (ms): 219482.1 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 76439.9 | valid tokens/s: 76439.9 | physical tokens/s: 77022.4 | loss tokens/s: 58329.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.654578E-01 | lm-global-supervised-tokens: 1.280219E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.353360E-02 | lm-token-buffer-mib: 2.341018E+02 | global_load_balancing_loss: 1.000688E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11191.734 | params rms: 0.060600 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:52:39.459682 | finish at 2026-10-17 16:52:47 + [2026-10-13 21:03:49] iteration 878/ 2384 | consumed samples: 28096 | elapsed time per iteration (ms): 220734.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 76006.1 | valid tokens/s: 76006.1 | physical tokens/s: 76571.1 | loss tokens/s: 54942.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.954958E-01 | lm-global-supervised-tokens: 1.212768E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.177407E-02 | lm-token-buffer-mib: 2.329806E+02 | global_load_balancing_loss: 1.000210E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11191.693 | params rms: 0.060599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:20:26.385976 | finish at 2026-10-17 17:24:15 + [2026-10-13 21:07:32] iteration 879/ 2384 | consumed samples: 28128 | elapsed time per iteration (ms): 223724.9 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.72% | FLOPs estimator: workload-aware | logical tokens/s: 74990.3 | valid tokens/s: 74990.3 | physical tokens/s: 75340.8 | loss tokens/s: 55367.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.831994E-01 | lm-global-supervised-tokens: 1.238699E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.204123E-02 | lm-token-buffer-mib: 2.330636E+02 | global_load_balancing_loss: 1.002308E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11191.658 | params rms: 0.060599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:31:45.978099 | finish at 2026-10-17 18:39:18 + [2026-10-13 21:11:40] iteration 880/ 2384 | consumed samples: 28160 | elapsed time per iteration (ms): 247426.0 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 67806.9 | valid tokens/s: 67806.9 | physical tokens/s: 68165.8 | loss tokens/s: 48595.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.823836E-01 | lm-global-supervised-tokens: 1.202382E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.746943E-02 | lm-token-buffer-mib: 2.324938E+02 | global_load_balancing_loss: 1.001618E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11191.627 | params rms: 0.060599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:22:08.695572 | finish at 2026-10-18 04:33:48 +times across ranks (ms): + save-checkpoint: + rank 0: 25768.37 + rank 1: 25768.38 + rank 2: 25768.38 + rank 3: 25768.41 + rank 4: 25768.37 + rank 5: 25768.37 + rank 6: 25768.40 + rank 7: 25768.37 + rank 8: 25768.11 + rank 9: 25768.10 + rank 10: 25768.10 + rank 11: 25768.07 + rank 12: 25768.08 + rank 13: 25768.05 + rank 14: 25768.10 + rank 15: 25768.08 + rank 16: 25768.24 + rank 17: 25768.22 + rank 18: 25768.24 + rank 19: 25768.23 + rank 20: 25768.22 + rank 21: 25768.24 + rank 22: 25768.23 + rank 23: 25768.06 + rank 24: 25768.59 + rank 25: 25768.59 + rank 26: 25768.57 + rank 27: 25768.57 + rank 28: 25768.57 + rank 29: 25768.53 + rank 30: 25768.61 + rank 31: 25768.55 + rank 32: 25768.88 + rank 33: 25768.94 + rank 34: 25768.99 + rank 35: 25768.91 + rank 36: 25768.85 + rank 37: 25768.89 + rank 38: 25768.98 + rank 39: 25768.96 + rank 40: 25767.82 + rank 41: 25767.77 + rank 42: 25767.77 + rank 43: 25767.78 + rank 44: 25767.76 + rank 45: 25767.76 + rank 46: 25767.76 + rank 47: 25767.76 + rank 48: 25767.84 + rank 49: 25767.87 + rank 50: 25767.84 + rank 51: 25767.87 + rank 52: 25767.93 + rank 53: 25767.95 + rank 54: 25767.78 + rank 55: 25767.90 + rank 56: 25768.51 + rank 57: 25768.53 + rank 58: 25768.53 + rank 59: 25768.70 + rank 60: 25768.53 + rank 61: 25768.50 + rank 62: 25768.50 + rank 63: 25768.53 + [2026-10-13 21:15:45] iteration 881/ 2384 | consumed samples: 28192 | elapsed time per iteration (ms): 219061.6 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 76586.6 | valid tokens/s: 76586.6 | physical tokens/s: 77380.5 | loss tokens/s: 54484.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.728944E-01 | lm-global-supervised-tokens: 1.193550E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.963499E-02 | lm-token-buffer-mib: 2.339630E+02 | global_load_balancing_loss: 1.000227E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11191.593 | params rms: 0.060599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:27:29.595147 | finish at 2026-10-17 16:43:14 + failed to log checkpoint /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0000880 as wandb artifact: Unable to connect to server to verify API token. + [2026-10-13 21:19:37] iteration 882/ 2384 | consumed samples: 28224 | elapsed time per iteration (ms): 232640.8 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 72116.2 | valid tokens/s: 72116.2 | physical tokens/s: 73023.4 | loss tokens/s: 55000.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.867498E-01 | lm-global-supervised-tokens: 1.279532E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.490333E-02 | lm-token-buffer-mib: 2.326576E+02 | global_load_balancing_loss: 9.996476E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11191.565 | params rms: 0.060599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:03:46.544267 | finish at 2026-10-17 22:23:24 + [2026-10-13 21:23:17] iteration 883/ 2384 | consumed samples: 28256 | elapsed time per iteration (ms): 219815.6 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 76324.0 | valid tokens/s: 76324.0 | physical tokens/s: 77026.6 | loss tokens/s: 54367.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.406625E-01 | lm-global-supervised-tokens: 1.195082E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.341392E-02 | lm-token-buffer-mib: 2.335660E+02 | global_load_balancing_loss: 9.994037E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11191.540 | params rms: 0.060599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:39:03.177544 | finish at 2026-10-17 17:02:20 + [2026-10-13 21:27:00] iteration 884/ 2384 | consumed samples: 28288 | elapsed time per iteration (ms): 222898.2 | throughput per GPU (TFLOP/s/GPU): 26.4 | MFU 8.47% | FLOPs estimator: workload-aware | logical tokens/s: 75268.5 | valid tokens/s: 75268.5 | physical tokens/s: 75922.9 | loss tokens/s: 55467.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.903709E-01 | lm-global-supervised-tokens: 1.236362E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.516680E-02 | lm-token-buffer-mib: 2.357818E+02 | global_load_balancing_loss: 1.000039E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11191.516 | params rms: 0.060598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:52:27.282171 | finish at 2026-10-17 18:19:27 + [2026-10-13 21:30:41] iteration 885/ 2384 | consumed samples: 28320 | elapsed time per iteration (ms): 221123.7 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 75872.4 | valid tokens/s: 75872.4 | physical tokens/s: 76601.3 | loss tokens/s: 55484.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.253933E-01 | lm-global-supervised-tokens: 1.226901E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.400611E-02 | lm-token-buffer-mib: 2.339400E+02 | global_load_balancing_loss: 9.997057E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11191.489 | params rms: 0.060598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:04:24.419722 | finish at 2026-10-17 17:35:06 + [2026-10-13 21:34:24] iteration 886/ 2384 | consumed samples: 28352 | elapsed time per iteration (ms): 223018.3 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 75227.9 | valid tokens/s: 75227.9 | physical tokens/s: 76226.1 | loss tokens/s: 61157.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.875844E-01 | lm-global-supervised-tokens: 1.363932E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.749225E-02 | lm-token-buffer-mib: 2.338306E+02 | global_load_balancing_loss: 1.000563E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11191.460 | params rms: 0.060598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:48:01.466343 | finish at 2026-10-17 18:22:26 + [2026-10-13 21:38:06] iteration 887/ 2384 | consumed samples: 28384 | elapsed time per iteration (ms): 221477.6 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.48% | FLOPs estimator: workload-aware | logical tokens/s: 75751.3 | valid tokens/s: 75751.3 | physical tokens/s: 76128.4 | loss tokens/s: 58364.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.633123E-01 | lm-global-supervised-tokens: 1.292645E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.314855E-02 | lm-token-buffer-mib: 2.327530E+02 | global_load_balancing_loss: 1.000245E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11191.433 | params rms: 0.060598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:05:51.991973 | finish at 2026-10-17 17:43:58 + [2026-10-13 21:41:48] iteration 888/ 2384 | consumed samples: 28416 | elapsed time per iteration (ms): 222607.5 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.48% | FLOPs estimator: workload-aware | logical tokens/s: 75366.7 | valid tokens/s: 75366.7 | physical tokens/s: 76314.7 | loss tokens/s: 58389.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.242359E-01 | lm-global-supervised-tokens: 1.299796E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.869969E-02 | lm-token-buffer-mib: 2.346669E+02 | global_load_balancing_loss: 1.001881E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11191.405 | params rms: 0.060598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:30:20.883602 | finish at 2026-10-17 18:12:09 + [2026-10-13 21:45:32] iteration 889/ 2384 | consumed samples: 28448 | elapsed time per iteration (ms): 223998.9 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 74898.6 | valid tokens/s: 74898.6 | physical tokens/s: 75499.2 | loss tokens/s: 59954.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.802638E-01 | lm-global-supervised-tokens: 1.342966E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.331657E-02 | lm-token-buffer-mib: 2.344042E+02 | global_load_balancing_loss: 1.000862E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11191.382 | params rms: 0.060598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:01:18.335088 | finish at 2026-10-17 18:46:51 + [2026-10-13 21:49:39] iteration 890/ 2384 | consumed samples: 28480 | elapsed time per iteration (ms): 246915.3 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 67947.2 | valid tokens/s: 67947.2 | physical tokens/s: 68632.1 | loss tokens/s: 46628.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.809579E-01 | lm-global-supervised-tokens: 1.151336E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.678344E-02 | lm-token-buffer-mib: 2.352833E+02 | global_load_balancing_loss: 1.000590E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11191.356 | params rms: 0.060598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:28:11.406747 | finish at 2026-10-18 04:17:51 + [2026-10-13 21:53:21] iteration 891/ 2384 | consumed samples: 28512 | elapsed time per iteration (ms): 222359.2 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75450.8 | valid tokens/s: 75450.8 | physical tokens/s: 75902.3 | loss tokens/s: 53091.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.043420E-01 | lm-global-supervised-tokens: 1.180528E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.518803E-02 | lm-token-buffer-mib: 2.337395E+02 | global_load_balancing_loss: 1.000935E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11191.326 | params rms: 0.060597 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:13:02.351810 | finish at 2026-10-17 18:06:24 + [2026-10-13 21:57:09] iteration 892/ 2384 | consumed samples: 28544 | elapsed time per iteration (ms): 227693.1 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.01% | FLOPs estimator: workload-aware | logical tokens/s: 73683.3 | valid tokens/s: 73683.3 | physical tokens/s: 74296.5 | loss tokens/s: 55706.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.177171E-01 | lm-global-supervised-tokens: 1.268402E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.509786E-02 | lm-token-buffer-mib: 2.344335E+02 | global_load_balancing_loss: 1.001158E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11191.294 | params rms: 0.060597 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:21:58.083465 | finish at 2026-10-17 20:19:07 + [2026-10-13 22:00:54] iteration 893/ 2384 | consumed samples: 28576 | elapsed time per iteration (ms): 224795.5 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 74633.2 | valid tokens/s: 74633.2 | physical tokens/s: 75335.3 | loss tokens/s: 56942.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.506409E-01 | lm-global-supervised-tokens: 1.280050E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.532542E-02 | lm-token-buffer-mib: 2.351598E+02 | global_load_balancing_loss: 9.997296E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11191.260 | params rms: 0.060597 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:06:10.037237 | finish at 2026-10-17 19:07:04 + [2026-10-13 22:04:35] iteration 894/ 2384 | consumed samples: 28608 | elapsed time per iteration (ms): 221076.0 | throughput per GPU (TFLOP/s/GPU): 17.9 | MFU 5.74% | FLOPs estimator: workload-aware | logical tokens/s: 75888.7 | valid tokens/s: 75888.7 | physical tokens/s: 76608.3 | loss tokens/s: 58471.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.922763E-01 | lm-global-supervised-tokens: 1.292655E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.012316E-02 | lm-token-buffer-mib: 2.335753E+02 | global_load_balancing_loss: 9.999449E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11191.228 | params rms: 0.060597 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:30:03.271935 | finish at 2026-10-17 17:34:38 + [2026-10-13 22:08:20] iteration 895/ 2384 | consumed samples: 28640 | elapsed time per iteration (ms): 224795.3 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.67% | FLOPs estimator: workload-aware | logical tokens/s: 74633.2 | valid tokens/s: 74633.2 | physical tokens/s: 75357.0 | loss tokens/s: 58176.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.849827E-01 | lm-global-supervised-tokens: 1.307779E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.365330E-02 | lm-token-buffer-mib: 2.336856E+02 | global_load_balancing_loss: 9.999200E-01 | loss scale: 1.0 | grad norm: 0.233 | params norm: 11191.195 | params rms: 0.060597 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:58:40.187865 | finish at 2026-10-17 19:07:00 + [2026-10-13 22:12:08] iteration 896/ 2384 | consumed samples: 28672 | elapsed time per iteration (ms): 227958.7 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.67% | FLOPs estimator: workload-aware | logical tokens/s: 73597.5 | valid tokens/s: 73597.5 | physical tokens/s: 75256.0 | loss tokens/s: 55664.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.401963E-01 | lm-global-supervised-tokens: 1.268928E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.517954E-02 | lm-token-buffer-mib: 2.381703E+02 | global_load_balancing_loss: 1.000949E+00 | loss scale: 1.0 | grad norm: 0.181 | params norm: 11191.156 | params rms: 0.060596 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:13:22.527065 | finish at 2026-10-17 20:25:30 + [2026-10-13 22:15:48] iteration 897/ 2384 | consumed samples: 28704 | elapsed time per iteration (ms): 219989.5 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 76263.6 | valid tokens/s: 76263.6 | physical tokens/s: 76671.6 | loss tokens/s: 54554.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.692822E-01 | lm-global-supervised-tokens: 1.200152E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.326327E-02 | lm-token-buffer-mib: 2.329718E+02 | global_load_balancing_loss: 1.001390E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11191.106 | params rms: 0.060596 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:52:04.452510 | finish at 2026-10-17 17:07:52 + [2026-10-13 22:19:28] iteration 898/ 2384 | consumed samples: 28736 | elapsed time per iteration (ms): 220676.5 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 76026.2 | valid tokens/s: 76026.2 | physical tokens/s: 76577.9 | loss tokens/s: 59344.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.831176E-01 | lm-global-supervised-tokens: 1.309602E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.688940E-02 | lm-token-buffer-mib: 2.333021E+02 | global_load_balancing_loss: 9.998631E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11191.059 | params rms: 0.060596 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:05:25.295419 | finish at 2026-10-17 17:24:54 + [2026-10-13 22:23:12] iteration 899/ 2384 | consumed samples: 28768 | elapsed time per iteration (ms): 223798.9 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 74965.4 | valid tokens/s: 74965.4 | physical tokens/s: 75698.0 | loss tokens/s: 57663.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.693665E-01 | lm-global-supervised-tokens: 1.290506E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.084921E-02 | lm-token-buffer-mib: 2.352783E+02 | global_load_balancing_loss: 1.000371E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11191.017 | params rms: 0.060596 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:19:01.312519 | finish at 2026-10-17 18:42:14 + [2026-10-13 22:27:11] iteration 900/ 2384 | consumed samples: 28800 | elapsed time per iteration (ms): 239047.6 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 70183.5 | valid tokens/s: 70183.5 | physical tokens/s: 70711.5 | loss tokens/s: 54244.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.546417E-01 | lm-global-supervised-tokens: 1.296706E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.385758E-02 | lm-token-buffer-mib: 2.332767E+02 | global_load_balancing_loss: 1.000091E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11190.979 | params rms: 0.060596 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:32:26.712393 | finish at 2026-10-18 00:59:38 + [2026-10-13 22:30:51] iteration 901/ 2384 | consumed samples: 28832 | elapsed time per iteration (ms): 220080.9 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 76231.9 | valid tokens/s: 76231.9 | physical tokens/s: 76760.7 | loss tokens/s: 58729.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.908841E-01 | lm-global-supervised-tokens: 1.292521E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.002482E-02 | lm-token-buffer-mib: 2.336014E+02 | global_load_balancing_loss: 9.990818E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11190.943 | params rms: 0.060595 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:39:39.916645 | finish at 2026-10-17 17:10:31 + [2026-10-13 22:34:34] iteration 902/ 2384 | consumed samples: 28864 | elapsed time per iteration (ms): 222286.2 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 75475.7 | valid tokens/s: 75475.7 | physical tokens/s: 76405.7 | loss tokens/s: 60213.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.827168E-01 | lm-global-supervised-tokens: 1.338462E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.762883E-02 | lm-token-buffer-mib: 2.339707E+02 | global_load_balancing_loss: 1.000978E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11190.908 | params rms: 0.060595 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:30:28.154122 | finish at 2026-10-17 18:05:02 + [2026-10-13 22:38:17] iteration 903/ 2384 | consumed samples: 28896 | elapsed time per iteration (ms): 223700.2 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 74998.5 | valid tokens/s: 74998.5 | physical tokens/s: 75459.0 | loss tokens/s: 53985.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.917140E-01 | lm-global-supervised-tokens: 1.207662E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.701684E-02 | lm-token-buffer-mib: 2.335515E+02 | global_load_balancing_loss: 1.000405E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11190.871 | params rms: 0.060595 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:01:40.043988 | finish at 2026-10-17 18:39:57 + [2026-10-13 22:42:04] iteration 904/ 2384 | consumed samples: 28928 | elapsed time per iteration (ms): 226154.3 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 74184.6 | valid tokens/s: 74184.6 | physical tokens/s: 74998.7 | loss tokens/s: 57352.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.416190E-01 | lm-global-supervised-tokens: 1.297040E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.446326E-02 | lm-token-buffer-mib: 2.353644E+02 | global_load_balancing_loss: 9.998409E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11190.836 | params rms: 0.060595 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:58:28.392544 | finish at 2026-10-17 19:40:32 + [2026-10-13 22:45:50] iteration 905/ 2384 | consumed samples: 28960 | elapsed time per iteration (ms): 226161.9 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 74182.3 | valid tokens/s: 74182.3 | physical tokens/s: 74956.3 | loss tokens/s: 56632.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.461923E-01 | lm-global-supervised-tokens: 1.280809E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.228195E-02 | lm-token-buffer-mib: 2.343824E+02 | global_load_balancing_loss: 1.000365E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11190.797 | params rms: 0.060595 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:54:53.513283 | finish at 2026-10-17 19:40:43 + [2026-10-13 22:49:35] iteration 906/ 2384 | consumed samples: 28992 | elapsed time per iteration (ms): 225127.2 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.86% | FLOPs estimator: workload-aware | logical tokens/s: 74523.1 | valid tokens/s: 74523.1 | physical tokens/s: 75199.0 | loss tokens/s: 60275.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.464370E-01 | lm-global-supervised-tokens: 1.356956E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.305378E-02 | lm-token-buffer-mib: 2.346247E+02 | global_load_balancing_loss: 9.993325E-01 | loss scale: 1.0 | grad norm: 0.199 | params norm: 11190.754 | params rms: 0.060594 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:25:38.040197 | finish at 2026-10-17 19:15:13 + [2026-10-13 22:53:20] iteration 907/ 2384 | consumed samples: 29024 | elapsed time per iteration (ms): 225094.6 | throughput per GPU (TFLOP/s/GPU): 26.7 | MFU 8.56% | FLOPs estimator: workload-aware | logical tokens/s: 74534.0 | valid tokens/s: 74534.0 | physical tokens/s: 74791.7 | loss tokens/s: 52553.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.560569E-01 | lm-global-supervised-tokens: 1.182942E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.550317E-02 | lm-token-buffer-mib: 2.329929E+02 | global_load_balancing_loss: 1.000067E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11190.713 | params rms: 0.060594 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:21:04.785769 | finish at 2026-10-17 19:14:25 + [2026-10-13 22:57:07] iteration 908/ 2384 | consumed samples: 29056 | elapsed time per iteration (ms): 227276.2 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 73818.6 | valid tokens/s: 73818.5 | physical tokens/s: 74927.3 | loss tokens/s: 56344.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.102323E-01 | lm-global-supervised-tokens: 1.280571E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.247702E-02 | lm-token-buffer-mib: 2.355745E+02 | global_load_balancing_loss: 1.000489E+00 | loss scale: 1.0 | grad norm: 0.247 | params norm: 11190.669 | params rms: 0.060594 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:10:59.661077 | finish at 2026-10-17 20:08:07 + [2026-10-13 23:00:52] iteration 909/ 2384 | consumed samples: 29088 | elapsed time per iteration (ms): 225072.9 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 74541.1 | valid tokens/s: 74541.1 | physical tokens/s: 75650.5 | loss tokens/s: 56970.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.883194E-01 | lm-global-supervised-tokens: 1.282259E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.343048E-02 | lm-token-buffer-mib: 2.369050E+02 | global_load_balancing_loss: 1.000731E+00 | loss scale: 1.0 | grad norm: 0.608 | params norm: 11190.626 | params rms: 0.060594 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:13:02.456899 | finish at 2026-10-17 19:13:55 + [2026-10-13 23:04:53] iteration 910/ 2384 | consumed samples: 29120 | elapsed time per iteration (ms): 241144.5 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 69573.2 | valid tokens/s: 69573.2 | physical tokens/s: 70149.7 | loss tokens/s: 54946.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.297908E-01 | lm-global-supervised-tokens: 1.325004E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.645217E-02 | lm-token-buffer-mib: 2.337142E+02 | global_load_balancing_loss: 1.001436E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11190.579 | params rms: 0.060593 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:44:07.002162 | finish at 2026-10-18 01:49:00 + [2026-10-13 23:08:41] iteration 911/ 2384 | consumed samples: 29152 | elapsed time per iteration (ms): 227443.3 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 73764.3 | valid tokens/s: 73764.3 | physical tokens/s: 74967.6 | loss tokens/s: 56330.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.587208E-01 | lm-global-supervised-tokens: 1.281209E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.333237E-02 | lm-token-buffer-mib: 2.383914E+02 | global_load_balancing_loss: 1.002516E+00 | loss scale: 1.0 | grad norm: 0.935 | params norm: 11190.534 | params rms: 0.060593 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:03:43.943336 | finish at 2026-10-17 20:12:25 + [2026-10-13 23:12:23] iteration 912/ 2384 | consumed samples: 29184 | elapsed time per iteration (ms): 222524.6 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 75394.7 | valid tokens/s: 75394.7 | physical tokens/s: 75912.0 | loss tokens/s: 55507.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.073929E-01 | lm-global-supervised-tokens: 1.235167E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.253162E-02 | lm-token-buffer-mib: 2.339031E+02 | global_load_balancing_loss: 1.000428E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11190.484 | params rms: 0.060593 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:59:16.283890 | finish at 2026-10-17 18:11:40 + [2026-10-13 23:16:06] iteration 913/ 2384 | consumed samples: 29216 | elapsed time per iteration (ms): 222544.4 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 75388.1 | valid tokens/s: 75388.1 | physical tokens/s: 75880.8 | loss tokens/s: 55575.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.458323E-01 | lm-global-supervised-tokens: 1.236792E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.991012E-02 | lm-token-buffer-mib: 2.340885E+02 | global_load_balancing_loss: 1.001231E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11190.439 | params rms: 0.060593 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:56:02.867077 | finish at 2026-10-17 18:12:09 + [2026-10-13 23:19:48] iteration 914/ 2384 | consumed samples: 29248 | elapsed time per iteration (ms): 222270.0 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 75481.2 | valid tokens/s: 75481.2 | physical tokens/s: 76088.2 | loss tokens/s: 58993.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.262563E-01 | lm-global-supervised-tokens: 1.311238E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.357074E-02 | lm-token-buffer-mib: 2.339646E+02 | global_load_balancing_loss: 1.000524E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11190.398 | params rms: 0.060592 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:45:36.874037 | finish at 2026-10-17 18:05:25 + [2026-10-13 23:23:31] iteration 915/ 2384 | consumed samples: 29280 | elapsed time per iteration (ms): 222930.3 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.01% | FLOPs estimator: workload-aware | logical tokens/s: 75257.4 | valid tokens/s: 75257.4 | physical tokens/s: 75762.4 | loss tokens/s: 56793.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.351452E-01 | lm-global-supervised-tokens: 1.266108E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.992407E-02 | lm-token-buffer-mib: 2.330852E+02 | global_load_balancing_loss: 1.000125E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11190.361 | params rms: 0.060592 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:58:04.668835 | finish at 2026-10-17 18:21:36 + [2026-10-13 23:27:12] iteration 916/ 2384 | consumed samples: 29312 | elapsed time per iteration (ms): 221256.2 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 75826.9 | valid tokens/s: 75826.9 | physical tokens/s: 76683.3 | loss tokens/s: 57549.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.360802E-01 | lm-global-supervised-tokens: 1.273307E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.312842E-02 | lm-token-buffer-mib: 2.335447E+02 | global_load_balancing_loss: 9.981647E-01 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11190.321 | params rms: 0.060592 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:13:24.106610 | finish at 2026-10-17 17:40:36 + [2026-10-13 23:30:53] iteration 917/ 2384 | consumed samples: 29344 | elapsed time per iteration (ms): 220940.9 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 75935.3 | valid tokens/s: 75935.3 | physical tokens/s: 76668.2 | loss tokens/s: 53550.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.019585E-01 | lm-global-supervised-tokens: 1.183143E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.305704E-02 | lm-token-buffer-mib: 2.336106E+02 | global_load_balancing_loss: 9.989596E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11190.281 | params rms: 0.060592 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:02:00.284689 | finish at 2026-10-17 17:32:54 + [2026-10-13 23:34:40] iteration 918/ 2384 | consumed samples: 29376 | elapsed time per iteration (ms): 226541.8 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 74057.8 | valid tokens/s: 74057.8 | physical tokens/s: 74978.6 | loss tokens/s: 55516.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.188198E-01 | lm-global-supervised-tokens: 1.257690E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.642450E-02 | lm-token-buffer-mib: 2.355006E+02 | global_load_balancing_loss: 9.992278E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11190.240 | params rms: 0.060592 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:15:10.265900 | finish at 2026-10-17 19:49:50 + [2026-10-13 23:38:22] iteration 919/ 2384 | consumed samples: 29408 | elapsed time per iteration (ms): 222017.2 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 75567.2 | valid tokens/s: 75567.2 | physical tokens/s: 76431.7 | loss tokens/s: 54620.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.495723E-01 | lm-global-supervised-tokens: 1.212673E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.792304E-02 | lm-token-buffer-mib: 2.335772E+02 | global_load_balancing_loss: 1.000587E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11190.199 | params rms: 0.060591 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:20:55.213708 | finish at 2026-10-17 17:59:17 + [2026-10-13 23:42:24] iteration 920/ 2384 | consumed samples: 29440 | elapsed time per iteration (ms): 241869.1 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.23% | FLOPs estimator: workload-aware | logical tokens/s: 69364.8 | valid tokens/s: 69364.8 | physical tokens/s: 69831.9 | loss tokens/s: 50063.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.570616E-01 | lm-global-supervised-tokens: 1.210885E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.635202E-02 | lm-token-buffer-mib: 2.330967E+02 | global_load_balancing_loss: 1.002257E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11190.158 | params rms: 0.060591 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:21:36.335661 | finish at 2026-10-18 02:04:00 +times across ranks (ms): + save-checkpoint: + rank 0: 27491.85 + rank 1: 27491.89 + rank 2: 27491.86 + rank 3: 27491.86 + rank 4: 27491.68 + rank 5: 27491.89 + rank 6: 27491.88 + rank 7: 27491.89 + rank 8: 27491.99 + rank 9: 27492.02 + rank 10: 27491.94 + rank 11: 27491.92 + rank 12: 27491.93 + rank 13: 27491.93 + rank 14: 27491.96 + rank 15: 27491.93 + rank 16: 27492.01 + rank 17: 27492.00 + rank 18: 27492.05 + rank 19: 27492.01 + rank 20: 27491.98 + rank 21: 27492.00 + rank 22: 27492.03 + rank 23: 27491.99 + rank 24: 27492.13 + rank 25: 27492.19 + rank 26: 27492.07 + rank 27: 27492.68 + rank 28: 27492.10 + rank 29: 27492.11 + rank 30: 27492.13 + rank 31: 27492.09 + rank 32: 27491.80 + rank 33: 27491.79 + rank 34: 27491.80 + rank 35: 27491.84 + rank 36: 27491.70 + rank 37: 27491.78 + rank 38: 27491.71 + rank 39: 27491.68 + rank 40: 27492.13 + rank 41: 27492.02 + rank 42: 27492.01 + rank 43: 27491.05 + rank 44: 27491.99 + rank 45: 27492.01 + rank 46: 27491.99 + rank 47: 27492.11 + rank 48: 27491.85 + rank 49: 27491.87 + rank 50: 27491.79 + rank 51: 27491.88 + rank 52: 27491.84 + rank 53: 27491.89 + rank 54: 27491.83 + rank 55: 27491.87 + rank 56: 27492.03 + rank 57: 27492.12 + rank 58: 27491.98 + rank 59: 27492.07 + rank 60: 27492.04 + rank 61: 27491.97 + rank 62: 27492.02 + rank 63: 27492.04 + [2026-10-13 23:46:59] iteration 921/ 2384 | consumed samples: 29472 | elapsed time per iteration (ms): 247878.7 | throughput per GPU (TFLOP/s/GPU): 16.3 | MFU 5.23% | FLOPs estimator: workload-aware | logical tokens/s: 67683.1 | valid tokens/s: 67683.1 | physical tokens/s: 68768.6 | loss tokens/s: 53037.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.286181E-01 | lm-global-supervised-tokens: 1.314674E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.964771E+00 | lm-token-buffer-mib: 2.369971E+02 | global_load_balancing_loss: 1.002300E+00 | loss scale: 1.0 | grad norm: 0.202 | params norm: 11190.119 | params rms: 0.060591 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:44:06.591145 | finish at 2026-10-18 04:31:06 + failed to log checkpoint /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0000920 as wandb artifact: Unable to connect to server to verify API token. + [2026-10-13 23:50:58] iteration 922/ 2384 | consumed samples: 29504 | elapsed time per iteration (ms): 238774.4 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 70263.7 | valid tokens/s: 70263.7 | physical tokens/s: 71017.6 | loss tokens/s: 54039.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.326998E-01 | lm-global-supervised-tokens: 1.290322E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.318316E-01 | lm-token-buffer-mib: 2.349522E+02 | global_load_balancing_loss: 1.001323E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11190.079 | params rms: 0.060591 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:58:08.196496 | finish at 2026-10-18 00:49:06 + [2026-10-13 23:54:39] iteration 923/ 2384 | consumed samples: 29536 | elapsed time per iteration (ms): 220727.4 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 76008.7 | valid tokens/s: 76008.7 | physical tokens/s: 76492.7 | loss tokens/s: 57631.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.105130E-01 | lm-global-supervised-tokens: 1.272086E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.081570E-02 | lm-token-buffer-mib: 2.329380E+02 | global_load_balancing_loss: 1.000870E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11190.035 | params rms: 0.060590 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:34:42.674368 | finish at 2026-10-17 17:29:21 + [2026-10-13 23:58:21] iteration 924/ 2384 | consumed samples: 29568 | elapsed time per iteration (ms): 222252.5 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 75487.0 | valid tokens/s: 75487.0 | physical tokens/s: 76407.5 | loss tokens/s: 59054.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.625440E-01 | lm-global-supervised-tokens: 1.312508E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.740819E-02 | lm-token-buffer-mib: 2.361848E+02 | global_load_balancing_loss: 9.999263E-01 | loss scale: 1.0 | grad norm: 0.295 | params norm: 11189.990 | params rms: 0.060590 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:08:08.605180 | finish at 2026-10-17 18:06:30 + [2026-10-14 00:02:03] iteration 925/ 2384 | consumed samples: 29600 | elapsed time per iteration (ms): 221673.7 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 75684.2 | valid tokens/s: 75684.2 | physical tokens/s: 76353.2 | loss tokens/s: 59230.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.836359E-01 | lm-global-supervised-tokens: 1.312974E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.540161E-02 | lm-token-buffer-mib: 2.334619E+02 | global_load_balancing_loss: 9.992596E-01 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11189.946 | params rms: 0.060590 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:50:21.908262 | finish at 2026-10-17 17:52:24 + [2026-10-14 00:05:50] iteration 926/ 2384 | consumed samples: 29632 | elapsed time per iteration (ms): 227802.9 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 73647.9 | valid tokens/s: 73647.8 | physical tokens/s: 75201.9 | loss tokens/s: 57747.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.311432E-01 | lm-global-supervised-tokens: 1.315514E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.402083E-02 | lm-token-buffer-mib: 2.368562E+02 | global_load_balancing_loss: 9.994730E-01 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11189.900 | params rms: 0.060590 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:15:36.557050 | finish at 2026-10-17 20:21:27 + [2026-10-14 00:09:30] iteration 927/ 2384 | consumed samples: 29664 | elapsed time per iteration (ms): 220094.3 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 76227.3 | valid tokens/s: 76227.2 | physical tokens/s: 76567.3 | loss tokens/s: 55669.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.956798E-01 | lm-global-supervised-tokens: 1.225251E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.283682E-02 | lm-token-buffer-mib: 2.329233E+02 | global_load_balancing_loss: 1.000067E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11189.853 | params rms: 0.060589 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:04:37.361798 | finish at 2026-10-17 17:14:08 + [2026-10-14 00:13:15] iteration 928/ 2384 | consumed samples: 29696 | elapsed time per iteration (ms): 224307.6 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.58% | FLOPs estimator: workload-aware | logical tokens/s: 74795.2 | valid tokens/s: 74795.2 | physical tokens/s: 75823.9 | loss tokens/s: 59087.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.695060E-01 | lm-global-supervised-tokens: 1.325386E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.786767E-02 | lm-token-buffer-mib: 2.341065E+02 | global_load_balancing_loss: 1.000713E+00 | loss scale: 1.0 | grad norm: 0.308 | params norm: 11189.807 | params rms: 0.060589 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:43:11.797592 | finish at 2026-10-17 18:56:27 + [2026-10-14 00:16:59] iteration 929/ 2384 | consumed samples: 29728 | elapsed time per iteration (ms): 224363.0 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.67% | FLOPs estimator: workload-aware | logical tokens/s: 74776.9 | valid tokens/s: 74776.9 | physical tokens/s: 75218.4 | loss tokens/s: 53915.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.266308E-01 | lm-global-supervised-tokens: 1.209652E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.970670E-02 | lm-token-buffer-mib: 2.338612E+02 | global_load_balancing_loss: 1.001091E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11189.759 | params rms: 0.060589 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:40:48.187079 | finish at 2026-10-17 18:57:47 + [2026-10-14 00:21:01] iteration 930/ 2384 | consumed samples: 29760 | elapsed time per iteration (ms): 241809.1 | throughput per GPU (TFLOP/s/GPU): 18.4 | MFU 5.89% | FLOPs estimator: workload-aware | logical tokens/s: 69382.1 | valid tokens/s: 69382.1 | physical tokens/s: 69944.0 | loss tokens/s: 52520.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.488166E-01 | lm-global-supervised-tokens: 1.269984E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.576408E-02 | lm-token-buffer-mib: 2.335607E+02 | global_load_balancing_loss: 9.994186E-01 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11189.707 | params rms: 0.060589 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:39:50.374074 | finish at 2026-10-18 02:00:51 + [2026-10-14 00:24:47] iteration 931/ 2384 | consumed samples: 29792 | elapsed time per iteration (ms): 225588.1 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 74370.8 | valid tokens/s: 74370.8 | physical tokens/s: 75022.7 | loss tokens/s: 56220.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.177348E-01 | lm-global-supervised-tokens: 1.268268E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.138128E-02 | lm-token-buffer-mib: 2.334521E+02 | global_load_balancing_loss: 9.998121E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11189.657 | params rms: 0.060588 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:02:59.532794 | finish at 2026-10-17 19:27:46 + [2026-10-14 00:28:28] iteration 932/ 2384 | consumed samples: 29824 | elapsed time per iteration (ms): 221454.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75759.2 | valid tokens/s: 75759.2 | physical tokens/s: 76571.0 | loss tokens/s: 58888.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.910605E-01 | lm-global-supervised-tokens: 1.304108E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.443879E-02 | lm-token-buffer-mib: 2.351939E+02 | global_load_balancing_loss: 9.994329E-01 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11189.606 | params rms: 0.060588 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:19:11.919402 | finish at 2026-10-17 17:47:40 + [2026-10-14 00:32:12] iteration 933/ 2384 | consumed samples: 29856 | elapsed time per iteration (ms): 224297.0 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.66% | FLOPs estimator: workload-aware | logical tokens/s: 74799.0 | valid tokens/s: 74799.0 | physical tokens/s: 75292.5 | loss tokens/s: 57825.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.958532E-01 | lm-global-supervised-tokens: 1.297013E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.517385E-02 | lm-token-buffer-mib: 2.337578E+02 | global_load_balancing_loss: 1.000897E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11189.555 | params rms: 0.060588 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:24:14.940327 | finish at 2026-10-17 18:56:27 + [2026-10-14 00:35:55] iteration 934/ 2384 | consumed samples: 29888 | elapsed time per iteration (ms): 222955.4 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75249.1 | valid tokens/s: 75249.1 | physical tokens/s: 75952.2 | loss tokens/s: 57252.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.389195E-01 | lm-global-supervised-tokens: 1.276474E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.324584E-02 | lm-token-buffer-mib: 2.327507E+02 | global_load_balancing_loss: 1.000335E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11189.502 | params rms: 0.060588 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:48:05.324454 | finish at 2026-10-17 18:24:01 + [2026-10-14 00:39:39] iteration 935/ 2384 | consumed samples: 29920 | elapsed time per iteration (ms): 223347.7 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 75117.0 | valid tokens/s: 75117.0 | physical tokens/s: 75642.6 | loss tokens/s: 60718.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.399305E-01 | lm-global-supervised-tokens: 1.356132E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.666839E-02 | lm-token-buffer-mib: 2.341403E+02 | global_load_balancing_loss: 1.001476E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11189.448 | params rms: 0.060587 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:53:50.840964 | finish at 2026-10-17 18:33:29 + [2026-10-14 00:43:21] iteration 936/ 2384 | consumed samples: 29952 | elapsed time per iteration (ms): 221932.4 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 75596.0 | valid tokens/s: 75596.0 | physical tokens/s: 76446.2 | loss tokens/s: 60030.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.995793E-01 | lm-global-supervised-tokens: 1.332269E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.231640E-02 | lm-token-buffer-mib: 2.331757E+02 | global_load_balancing_loss: 1.001417E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11189.393 | params rms: 0.060587 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:15:58.129683 | finish at 2026-10-17 17:59:19 + [2026-10-14 00:47:02] iteration 937/ 2384 | consumed samples: 29984 | elapsed time per iteration (ms): 221442.5 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.05% | FLOPs estimator: workload-aware | logical tokens/s: 75763.3 | valid tokens/s: 75763.3 | physical tokens/s: 77164.5 | loss tokens/s: 58325.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.968574E-01 | lm-global-supervised-tokens: 1.291572E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.873230E-02 | lm-token-buffer-mib: 2.348216E+02 | global_load_balancing_loss: 1.001896E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11189.339 | params rms: 0.060587 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:00:27.282831 | finish at 2026-10-17 17:47:29 + [2026-10-14 00:50:48] iteration 938/ 2384 | consumed samples: 30016 | elapsed time per iteration (ms): 226449.0 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 74088.2 | valid tokens/s: 74088.2 | physical tokens/s: 75073.5 | loss tokens/s: 53812.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.996346E-01 | lm-global-supervised-tokens: 1.218566E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.388886E-02 | lm-token-buffer-mib: 2.369635E+02 | global_load_balancing_loss: 9.990959E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11189.287 | params rms: 0.060586 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:57:25.259000 | finish at 2026-10-17 19:48:14 + [2026-10-14 00:54:32] iteration 939/ 2384 | consumed samples: 30048 | elapsed time per iteration (ms): 223152.9 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 75182.4 | valid tokens/s: 75182.4 | physical tokens/s: 75739.6 | loss tokens/s: 57235.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.468086E-01 | lm-global-supervised-tokens: 1.277219E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.599371E-02 | lm-token-buffer-mib: 2.342024E+02 | global_load_balancing_loss: 9.996639E-01 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11189.235 | params rms: 0.060586 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:34:16.004552 | finish at 2026-10-17 18:28:48 + [2026-10-14 00:58:34] iteration 940/ 2384 | consumed samples: 30080 | elapsed time per iteration (ms): 242860.1 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 69081.5 | valid tokens/s: 69081.5 | physical tokens/s: 69617.3 | loss tokens/s: 48705.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.849753E-01 | lm-global-supervised-tokens: 1.182863E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.635447E-02 | lm-token-buffer-mib: 2.331295E+02 | global_load_balancing_loss: 1.000539E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11189.179 | params rms: 0.060586 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:24:50.031610 | finish at 2026-10-18 02:23:24 + [2026-10-14 01:02:19] iteration 941/ 2384 | consumed samples: 30112 | elapsed time per iteration (ms): 224985.0 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 74570.3 | valid tokens/s: 74570.3 | physical tokens/s: 75320.8 | loss tokens/s: 55991.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.969975E-01 | lm-global-supervised-tokens: 1.259728E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.993397E-02 | lm-token-buffer-mib: 2.344351E+02 | global_load_balancing_loss: 1.001723E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11189.121 | params rms: 0.060585 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:10:53.407142 | finish at 2026-10-17 19:13:13 + [2026-10-14 01:06:01] iteration 942/ 2384 | consumed samples: 30144 | elapsed time per iteration (ms): 221677.3 | throughput per GPU (TFLOP/s/GPU): 26.9 | MFU 8.63% | FLOPs estimator: workload-aware | logical tokens/s: 75682.9 | valid tokens/s: 75682.9 | physical tokens/s: 75922.4 | loss tokens/s: 58589.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.581162E-01 | lm-global-supervised-tokens: 1.298800E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.254142E-02 | lm-token-buffer-mib: 2.324583E+02 | global_load_balancing_loss: 9.992710E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11189.066 | params rms: 0.060585 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:47:38.679974 | finish at 2026-10-17 17:53:40 + [2026-10-14 01:09:43] iteration 943/ 2384 | consumed samples: 30176 | elapsed time per iteration (ms): 221695.3 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.66% | FLOPs estimator: workload-aware | logical tokens/s: 75676.8 | valid tokens/s: 75676.8 | physical tokens/s: 76064.0 | loss tokens/s: 56555.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.526870E-01 | lm-global-supervised-tokens: 1.253817E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.500707E-02 | lm-token-buffer-mib: 2.329872E+02 | global_load_balancing_loss: 1.001007E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11189.014 | params rms: 0.060585 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:44:22.929509 | finish at 2026-10-17 17:54:06 + [2026-10-14 01:13:24] iteration 944/ 2384 | consumed samples: 30208 | elapsed time per iteration (ms): 221390.2 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 75780.9 | valid tokens/s: 75780.9 | physical tokens/s: 76406.8 | loss tokens/s: 57027.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.981419E-01 | lm-global-supervised-tokens: 1.262535E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.306503E-02 | lm-token-buffer-mib: 2.340311E+02 | global_load_balancing_loss: 1.001067E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11188.962 | params rms: 0.060585 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:33:21.934204 | finish at 2026-10-17 17:46:46 + [2026-10-14 01:17:08] iteration 945/ 2384 | consumed samples: 30240 | elapsed time per iteration (ms): 223493.5 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 75067.8 | valid tokens/s: 75067.8 | physical tokens/s: 75981.2 | loss tokens/s: 62196.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.872311E-01 | lm-global-supervised-tokens: 1.390053E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.401260E-02 | lm-token-buffer-mib: 2.347220E+02 | global_load_balancing_loss: 9.996409E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11188.911 | params rms: 0.060584 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:20:07.199327 | finish at 2026-10-17 18:37:15 + [2026-10-14 01:20:54] iteration 946/ 2384 | consumed samples: 30272 | elapsed time per iteration (ms): 225866.9 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 74279.0 | valid tokens/s: 74279.0 | physical tokens/s: 75176.0 | loss tokens/s: 54517.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.781358E-01 | lm-global-supervised-tokens: 1.231376E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.429176E-02 | lm-token-buffer-mib: 2.338056E+02 | global_load_balancing_loss: 9.999220E-01 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11188.862 | params rms: 0.060584 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:13:16.672779 | finish at 2026-10-17 19:34:10 + [2026-10-14 01:24:38] iteration 947/ 2384 | consumed samples: 30304 | elapsed time per iteration (ms): 224899.7 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.09% | FLOPs estimator: workload-aware | logical tokens/s: 74598.5 | valid tokens/s: 74598.5 | physical tokens/s: 75779.6 | loss tokens/s: 57541.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.693775E-01 | lm-global-supervised-tokens: 1.294111E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.774474E-02 | lm-token-buffer-mib: 2.360965E+02 | global_load_balancing_loss: 1.000148E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11188.811 | params rms: 0.060584 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:46:20.833506 | finish at 2026-10-17 19:10:59 + [2026-10-14 01:28:20] iteration 948/ 2384 | consumed samples: 30336 | elapsed time per iteration (ms): 221217.3 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 75840.3 | valid tokens/s: 75840.3 | physical tokens/s: 76675.4 | loss tokens/s: 57230.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.211838E-01 | lm-global-supervised-tokens: 1.266029E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.514222E-02 | lm-token-buffer-mib: 2.348951E+02 | global_load_balancing_loss: 1.001047E+00 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11188.758 | params rms: 0.060583 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:14:27.977319 | finish at 2026-10-17 17:42:48 + [2026-10-14 01:32:02] iteration 949/ 2384 | consumed samples: 30368 | elapsed time per iteration (ms): 222656.3 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 75350.1 | valid tokens/s: 75350.1 | physical tokens/s: 76044.2 | loss tokens/s: 56590.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.862845E-01 | lm-global-supervised-tokens: 1.260027E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.627495E-02 | lm-token-buffer-mib: 2.334266E+02 | global_load_balancing_loss: 1.001273E+00 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11188.706 | params rms: 0.060583 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:45:11.789913 | finish at 2026-10-17 18:17:14 + [2026-10-14 01:36:07] iteration 950/ 2384 | consumed samples: 30400 | elapsed time per iteration (ms): 244907.8 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.52% | FLOPs estimator: workload-aware | logical tokens/s: 68504.1 | valid tokens/s: 68504.1 | physical tokens/s: 69014.1 | loss tokens/s: 46671.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.392537E-01 | lm-global-supervised-tokens: 1.143026E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.127603E-02 | lm-token-buffer-mib: 2.339703E+02 | global_load_balancing_loss: 1.001337E+00 | loss scale: 1.0 | grad norm: 0.207 | params norm: 11188.652 | params rms: 0.060583 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:33:17.763260 | finish at 2026-10-18 03:09:25 + [2026-10-14 01:39:50] iteration 951/ 2384 | consumed samples: 30432 | elapsed time per iteration (ms): 222301.9 | throughput per GPU (TFLOP/s/GPU): 18.3 | MFU 5.87% | FLOPs estimator: workload-aware | logical tokens/s: 75470.3 | valid tokens/s: 75470.2 | physical tokens/s: 76410.7 | loss tokens/s: 58545.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.856326E-01 | lm-global-supervised-tokens: 1.301473E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.910176E-02 | lm-token-buffer-mib: 2.357096E+02 | global_load_balancing_loss: 1.000431E+00 | loss scale: 1.0 | grad norm: 0.243 | params norm: 11188.598 | params rms: 0.060583 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:29:18.599339 | finish at 2026-10-17 18:09:08 + [2026-10-14 01:43:29] iteration 952/ 2384 | consumed samples: 30464 | elapsed time per iteration (ms): 219642.4 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 76384.0 | valid tokens/s: 76384.0 | physical tokens/s: 77365.9 | loss tokens/s: 61896.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.071424E-01 | lm-global-supervised-tokens: 1.359511E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.912709E-02 | lm-token-buffer-mib: 2.335370E+02 | global_load_balancing_loss: 1.000179E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11188.548 | params rms: 0.060582 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:22:07.850603 | finish at 2026-10-17 17:05:37 + [2026-10-14 01:47:11] iteration 953/ 2384 | consumed samples: 30496 | elapsed time per iteration (ms): 221362.9 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 75790.4 | valid tokens/s: 75790.4 | physical tokens/s: 76339.3 | loss tokens/s: 54209.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.835454E-01 | lm-global-supervised-tokens: 1.199995E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.603588E-02 | lm-token-buffer-mib: 2.343965E+02 | global_load_balancing_loss: 9.998122E-01 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11188.496 | params rms: 0.060582 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:59:30.332786 | finish at 2026-10-17 17:46:41 + [2026-10-14 01:50:55] iteration 954/ 2384 | consumed samples: 30528 | elapsed time per iteration (ms): 224531.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 74721.0 | valid tokens/s: 74721.0 | physical tokens/s: 75565.1 | loss tokens/s: 57533.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.748905E-01 | lm-global-supervised-tokens: 1.291816E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.768963E-02 | lm-token-buffer-mib: 2.353913E+02 | global_load_balancing_loss: 9.989998E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11188.448 | params rms: 0.060582 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:11:19.527941 | finish at 2026-10-17 19:02:15 + [2026-10-14 01:54:38] iteration 955/ 2384 | consumed samples: 30560 | elapsed time per iteration (ms): 222777.9 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 75309.2 | valid tokens/s: 75309.2 | physical tokens/s: 76061.8 | loss tokens/s: 58083.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.482181E-01 | lm-global-supervised-tokens: 1.293973E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.284581E-02 | lm-token-buffer-mib: 2.343195E+02 | global_load_balancing_loss: 9.993736E-01 | loss scale: 1.0 | grad norm: 0.385 | params norm: 11188.400 | params rms: 0.060582 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:25:49.608170 | finish at 2026-10-17 18:20:27 + [2026-10-14 01:58:23] iteration 956/ 2384 | consumed samples: 30592 | elapsed time per iteration (ms): 224699.0 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 74665.2 | valid tokens/s: 74665.2 | physical tokens/s: 75448.8 | loss tokens/s: 53000.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.779208E-01 | lm-global-supervised-tokens: 1.190915E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.524312E-02 | lm-token-buffer-mib: 2.350768E+02 | global_load_balancing_loss: 1.000494E+00 | loss scale: 1.0 | grad norm: 0.229 | params norm: 11188.348 | params rms: 0.060581 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:07:50.216772 | finish at 2026-10-17 19:06:13 + [2026-10-14 02:02:02] iteration 957/ 2384 | consumed samples: 30624 | elapsed time per iteration (ms): 219610.7 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 76395.2 | valid tokens/s: 76395.2 | physical tokens/s: 76789.3 | loss tokens/s: 59052.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.670730E-01 | lm-global-supervised-tokens: 1.296857E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.269624E-02 | lm-token-buffer-mib: 2.330789E+02 | global_load_balancing_loss: 1.001611E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11188.296 | params rms: 0.060581 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:03:04.461601 | finish at 2026-10-17 17:05:07 + [2026-10-14 02:05:44] iteration 958/ 2384 | consumed samples: 30656 | elapsed time per iteration (ms): 221406.2 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 75775.7 | valid tokens/s: 75775.7 | physical tokens/s: 76585.4 | loss tokens/s: 58626.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.658139E-01 | lm-global-supervised-tokens: 1.298032E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.295907E-02 | lm-token-buffer-mib: 2.347370E+02 | global_load_balancing_loss: 1.002150E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11188.247 | params rms: 0.060581 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:42:05.278162 | finish at 2026-10-17 17:47:49 + [2026-10-14 02:09:27] iteration 959/ 2384 | consumed samples: 30688 | elapsed time per iteration (ms): 223849.6 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 74948.5 | valid tokens/s: 74948.5 | physical tokens/s: 75872.1 | loss tokens/s: 53174.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.344902E-01 | lm-global-supervised-tokens: 1.190301E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.552953E-02 | lm-token-buffer-mib: 2.343716E+02 | global_load_balancing_loss: 1.001353E+00 | loss scale: 1.0 | grad norm: 0.283 | params norm: 11188.201 | params rms: 0.060580 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:36:25.615218 | finish at 2026-10-17 18:45:53 + [2026-10-14 02:13:33] iteration 960/ 2384 | consumed samples: 30720 | elapsed time per iteration (ms): 245795.5 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 68256.7 | valid tokens/s: 68256.7 | physical tokens/s: 68898.9 | loss tokens/s: 54141.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.447038E-01 | lm-global-supervised-tokens: 1.330778E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.159541E-02 | lm-token-buffer-mib: 2.337471E+02 | global_load_balancing_loss: 1.000681E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11188.155 | params rms: 0.060580 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:13:32.783569 | finish at 2026-10-18 03:27:06 +times across ranks (ms): + save-checkpoint: + rank 0: 24954.80 + rank 1: 24954.71 + rank 2: 24954.76 + rank 3: 24954.75 + rank 4: 24954.73 + rank 5: 24954.75 + rank 6: 24954.79 + rank 7: 24954.72 + rank 8: 24953.88 + rank 9: 24953.93 + rank 10: 24953.94 + rank 11: 24953.90 + rank 12: 24953.89 + rank 13: 24953.87 + rank 14: 24953.92 + rank 15: 24953.88 + rank 16: 24954.43 + rank 17: 24954.38 + rank 18: 24954.38 + rank 19: 24954.37 + rank 20: 24954.36 + rank 21: 24954.35 + rank 22: 24954.40 + rank 23: 24954.32 + rank 24: 24954.97 + rank 25: 24954.95 + rank 26: 24954.93 + rank 27: 24955.13 + rank 28: 24954.94 + rank 29: 24954.95 + rank 30: 24954.98 + rank 31: 24954.93 + rank 32: 24954.87 + rank 33: 24954.85 + rank 34: 24954.87 + rank 35: 24954.92 + rank 36: 24954.82 + rank 37: 24954.92 + rank 38: 24954.88 + rank 39: 24954.77 + rank 40: 24954.56 + rank 41: 24954.71 + rank 42: 24954.71 + rank 43: 24954.69 + rank 44: 24954.71 + rank 45: 24954.71 + rank 46: 24954.71 + rank 47: 24954.71 + rank 48: 24954.66 + rank 49: 24954.82 + rank 50: 24954.71 + rank 51: 24954.91 + rank 52: 24954.69 + rank 53: 24954.83 + rank 54: 24954.68 + rank 55: 24954.65 + rank 56: 24954.97 + rank 57: 24954.96 + rank 58: 24954.98 + rank 59: 24954.97 + rank 60: 24954.97 + rank 61: 24954.97 + rank 62: 24954.96 + rank 63: 24954.96 + [2026-10-14 02:17:55] iteration 961/ 2384 | consumed samples: 30752 | elapsed time per iteration (ms): 236348.5 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.09% | FLOPs estimator: workload-aware | logical tokens/s: 70985.0 | valid tokens/s: 70985.0 | physical tokens/s: 72303.5 | loss tokens/s: 54584.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.465574E-01 | lm-global-supervised-tokens: 1.290090E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.604902E-01 | lm-token-buffer-mib: 2.366473E+02 | global_load_balancing_loss: 1.000365E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11188.109 | params rms: 0.060580 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:25:23.975570 | finish at 2026-10-17 23:43:19 + [2026-10-14 02:21:45] iteration 962/ 2384 | consumed samples: 30784 | elapsed time per iteration (ms): 230765.8 | throughput per GPU (TFLOP/s/GPU): 17.5 | MFU 5.61% | FLOPs estimator: workload-aware | logical tokens/s: 72702.4 | valid tokens/s: 72702.4 | physical tokens/s: 73829.5 | loss tokens/s: 55028.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.194007E-01 | lm-global-supervised-tokens: 1.269873E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.747158E-02 | lm-token-buffer-mib: 2.366097E+02 | global_load_balancing_loss: 9.999337E-01 | loss scale: 1.0 | grad norm: 0.225 | params norm: 11188.063 | params rms: 0.060580 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:09:08.925898 | finish at 2026-10-17 21:30:54 + [2026-10-14 02:25:26] iteration 963/ 2384 | consumed samples: 30816 | elapsed time per iteration (ms): 221093.8 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 75882.7 | valid tokens/s: 75882.7 | physical tokens/s: 76681.4 | loss tokens/s: 59493.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.671415E-01 | lm-global-supervised-tokens: 1.315365E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.281024E-02 | lm-token-buffer-mib: 2.350346E+02 | global_load_balancing_loss: 9.980396E-01 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11188.018 | params rms: 0.060579 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:16:14.304465 | finish at 2026-10-17 17:41:41 + [2026-10-14 02:29:07] iteration 964/ 2384 | consumed samples: 30848 | elapsed time per iteration (ms): 220516.5 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 76081.3 | valid tokens/s: 76081.3 | physical tokens/s: 76693.0 | loss tokens/s: 56531.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.576753E-01 | lm-global-supervised-tokens: 1.246612E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.712527E-02 | lm-token-buffer-mib: 2.337939E+02 | global_load_balancing_loss: 9.990905E-01 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11187.973 | params rms: 0.060579 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:58:53.471298 | finish at 2026-10-17 17:28:00 + [2026-10-14 02:32:48] iteration 965/ 2384 | consumed samples: 30880 | elapsed time per iteration (ms): 221143.1 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.65% | FLOPs estimator: workload-aware | logical tokens/s: 75865.8 | valid tokens/s: 75865.8 | physical tokens/s: 76587.9 | loss tokens/s: 57597.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.848649E-01 | lm-global-supervised-tokens: 1.273720E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.282414E-02 | lm-token-buffer-mib: 2.342774E+02 | global_load_balancing_loss: 9.995283E-01 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11187.927 | params rms: 0.060579 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:10:02.073142 | finish at 2026-10-17 17:42:50 + [2026-10-14 02:36:28] iteration 966/ 2384 | consumed samples: 30912 | elapsed time per iteration (ms): 219906.8 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 76292.2 | valid tokens/s: 76292.2 | physical tokens/s: 76891.9 | loss tokens/s: 57281.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.476559E-01 | lm-global-supervised-tokens: 1.259662E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.453116E-02 | lm-token-buffer-mib: 2.345425E+02 | global_load_balancing_loss: 1.001192E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11187.878 | params rms: 0.060579 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:37:07.831626 | finish at 2026-10-17 17:13:36 + [2026-10-14 02:40:13] iteration 967/ 2384 | consumed samples: 30944 | elapsed time per iteration (ms): 225361.5 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 74445.8 | valid tokens/s: 74445.8 | physical tokens/s: 75386.7 | loss tokens/s: 55477.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.564134E-01 | lm-global-supervised-tokens: 1.250259E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.334731E-02 | lm-token-buffer-mib: 2.373901E+02 | global_load_balancing_loss: 1.001539E+00 | loss scale: 1.0 | grad norm: 0.289 | params norm: 11187.833 | params rms: 0.060578 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:42:17.199927 | finish at 2026-10-17 19:22:31 + [2026-10-14 02:43:57] iteration 968/ 2384 | consumed samples: 30976 | elapsed time per iteration (ms): 223427.3 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75090.2 | valid tokens/s: 75090.2 | physical tokens/s: 75362.4 | loss tokens/s: 54501.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.977483E-01 | lm-global-supervised-tokens: 1.217707E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.323371E-02 | lm-token-buffer-mib: 2.328437E+02 | global_load_balancing_loss: 1.002529E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11187.792 | params rms: 0.060578 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:52:53.060143 | finish at 2026-10-17 18:36:50 + [2026-10-14 02:47:38] iteration 969/ 2384 | consumed samples: 31008 | elapsed time per iteration (ms): 221478.3 | throughput per GPU (TFLOP/s/GPU): 27.3 | MFU 8.74% | FLOPs estimator: workload-aware | logical tokens/s: 75750.9 | valid tokens/s: 75750.9 | physical tokens/s: 76243.4 | loss tokens/s: 56267.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.243350E-01 | lm-global-supervised-tokens: 1.246201E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.579664E-02 | lm-token-buffer-mib: 2.337478E+02 | global_load_balancing_loss: 1.001730E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11187.754 | params rms: 0.060578 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:03:11.864805 | finish at 2026-10-17 17:50:50 + [2026-10-14 02:51:42] iteration 970/ 2384 | consumed samples: 31040 | elapsed time per iteration (ms): 243300.7 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.32% | FLOPs estimator: workload-aware | logical tokens/s: 68956.6 | valid tokens/s: 68956.6 | physical tokens/s: 69712.2 | loss tokens/s: 52529.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.953471E-01 | lm-global-supervised-tokens: 1.278049E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.201477E-02 | lm-token-buffer-mib: 2.358062E+02 | global_load_balancing_loss: 9.992823E-01 | loss scale: 1.0 | grad norm: 0.282 | params norm: 11187.717 | params rms: 0.060578 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:33:47.187706 | finish at 2026-10-18 02:25:29 + [2026-10-14 02:55:22] iteration 971/ 2384 | consumed samples: 31072 | elapsed time per iteration (ms): 220109.7 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 76222.0 | valid tokens/s: 76222.0 | physical tokens/s: 76604.1 | loss tokens/s: 54372.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.560659E-01 | lm-global-supervised-tokens: 1.196780E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.561034E-02 | lm-token-buffer-mib: 2.325854E+02 | global_load_balancing_loss: 9.989921E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11187.677 | params rms: 0.060578 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:23:34.975056 | finish at 2026-10-17 17:18:57 + [2026-10-14 02:59:46] iteration 972/ 2384 | consumed samples: 31104 | elapsed time per iteration (ms): 264761.7 | throughput per GPU (TFLOP/s/GPU): 17.8 | MFU 5.71% | FLOPs estimator: workload-aware | logical tokens/s: 63367.2 | valid tokens/s: 63367.2 | physical tokens/s: 64946.2 | loss tokens/s: 46106.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.819838E-01 | lm-global-supervised-tokens: 1.220714E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.872401E-02 | lm-token-buffer-mib: 2.348354E+02 | global_load_balancing_loss: 1.000346E+00 | loss scale: 1.0 | grad norm: 0.302 | params norm: 11187.635 | params rms: 0.060577 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:50:43.464733 | finish at 2026-10-18 10:50:30 + [2026-10-14 03:03:29] iteration 973/ 2384 | consumed samples: 31136 | elapsed time per iteration (ms): 222860.3 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.32% | FLOPs estimator: workload-aware | logical tokens/s: 75281.2 | valid tokens/s: 75281.2 | physical tokens/s: 76799.9 | loss tokens/s: 58032.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.607485E-01 | lm-global-supervised-tokens: 1.293303E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.034507E-02 | lm-token-buffer-mib: 2.346877E+02 | global_load_balancing_loss: 9.988881E-01 | loss scale: 1.0 | grad norm: 0.267 | params norm: 11187.592 | params rms: 0.060577 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:20:55.903239 | finish at 2026-10-17 18:24:25 + [2026-10-14 03:07:11] iteration 974/ 2384 | consumed samples: 31168 | elapsed time per iteration (ms): 221820.8 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 75634.1 | valid tokens/s: 75634.1 | physical tokens/s: 76095.2 | loss tokens/s: 58942.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.954030E-01 | lm-global-supervised-tokens: 1.307464E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.610447E-02 | lm-token-buffer-mib: 2.338861E+02 | global_load_balancing_loss: 9.996139E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11187.553 | params rms: 0.060577 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:52:47.259178 | finish at 2026-10-17 17:59:58 + [2026-10-14 03:10:53] iteration 975/ 2384 | consumed samples: 31200 | elapsed time per iteration (ms): 222026.8 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.49% | FLOPs estimator: workload-aware | logical tokens/s: 75563.7 | valid tokens/s: 75563.7 | physical tokens/s: 76345.7 | loss tokens/s: 60898.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.505821E-01 | lm-global-supervised-tokens: 1.352102E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.401043E-02 | lm-token-buffer-mib: 2.337649E+02 | global_load_balancing_loss: 9.990566E-01 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11187.515 | params rms: 0.060577 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:53:55.738240 | finish at 2026-10-17 18:04:49 + [2026-10-14 03:14:39] iteration 976/ 2384 | consumed samples: 31232 | elapsed time per iteration (ms): 225577.4 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 74374.2 | valid tokens/s: 74374.2 | physical tokens/s: 75103.7 | loss tokens/s: 56011.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.220142E-01 | lm-global-supervised-tokens: 1.263490E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.438884E-02 | lm-token-buffer-mib: 2.347813E+02 | global_load_balancing_loss: 1.000093E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11187.476 | params rms: 0.060577 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:13:32.919739 | finish at 2026-10-17 19:28:12 + [2026-10-14 03:18:20] iteration 977/ 2384 | consumed samples: 31264 | elapsed time per iteration (ms): 221292.5 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 75814.6 | valid tokens/s: 75814.6 | physical tokens/s: 76850.3 | loss tokens/s: 59986.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.467901E-01 | lm-global-supervised-tokens: 1.327448E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.538828E-02 | lm-token-buffer-mib: 2.338451E+02 | global_load_balancing_loss: 1.003268E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11187.437 | params rms: 0.060576 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:29:18.590801 | finish at 2026-10-17 17:47:39 + [2026-10-14 03:22:02] iteration 978/ 2384 | consumed samples: 31296 | elapsed time per iteration (ms): 221916.6 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75601.4 | valid tokens/s: 75601.4 | physical tokens/s: 76169.9 | loss tokens/s: 56441.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.588662E-01 | lm-global-supervised-tokens: 1.252522E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.518699E-02 | lm-token-buffer-mib: 2.330396E+02 | global_load_balancing_loss: 1.002151E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11187.398 | params rms: 0.060576 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:40:14.672541 | finish at 2026-10-17 18:02:17 + [2026-10-14 03:25:42] iteration 979/ 2384 | consumed samples: 31328 | elapsed time per iteration (ms): 220383.1 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.93% | FLOPs estimator: workload-aware | logical tokens/s: 76127.4 | valid tokens/s: 76127.4 | physical tokens/s: 76677.0 | loss tokens/s: 54367.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.292534E-01 | lm-global-supervised-tokens: 1.198158E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.912393E-02 | lm-token-buffer-mib: 2.338107E+02 | global_load_balancing_loss: 1.000614E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11187.359 | params rms: 0.060576 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:00:38.197595 | finish at 2026-10-17 17:26:20 + [2026-10-14 03:29:44] iteration 980/ 2384 | consumed samples: 31360 | elapsed time per iteration (ms): 242204.4 | throughput per GPU (TFLOP/s/GPU): 18.3 | MFU 5.87% | FLOPs estimator: workload-aware | logical tokens/s: 69268.8 | valid tokens/s: 69268.8 | physical tokens/s: 69790.7 | loss tokens/s: 50166.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.939594E-01 | lm-global-supervised-tokens: 1.215065E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.671375E-02 | lm-token-buffer-mib: 2.334621E+02 | global_load_balancing_loss: 1.000986E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11187.316 | params rms: 0.060576 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:27:34.918774 | finish at 2026-10-18 01:57:19 + [2026-10-14 03:33:26] iteration 981/ 2384 | consumed samples: 31392 | elapsed time per iteration (ms): 221964.6 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 75585.0 | valid tokens/s: 75585.0 | physical tokens/s: 76361.6 | loss tokens/s: 57720.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.714535E-01 | lm-global-supervised-tokens: 1.281199E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.454137E-02 | lm-token-buffer-mib: 2.346338E+02 | global_load_balancing_loss: 1.000437E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11187.272 | params rms: 0.060575 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:30:16.283746 | finish at 2026-10-17 18:03:43 + [2026-10-14 03:37:15] iteration 982/ 2384 | consumed samples: 31424 | elapsed time per iteration (ms): 228056.3 | throughput per GPU (TFLOP/s/GPU): 26.3 | MFU 8.43% | FLOPs estimator: workload-aware | logical tokens/s: 73565.9 | valid tokens/s: 73565.9 | physical tokens/s: 73948.6 | loss tokens/s: 53467.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.762619E-01 | lm-global-supervised-tokens: 1.219370E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.588969E-02 | lm-token-buffer-mib: 2.329735E+02 | global_load_balancing_loss: 9.993302E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11187.230 | params rms: 0.060575 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:48:54.970601 | finish at 2026-10-17 20:26:09 + [2026-10-14 03:40:56] iteration 983/ 2384 | consumed samples: 31456 | elapsed time per iteration (ms): 221776.8 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 75648.9 | valid tokens/s: 75648.9 | physical tokens/s: 76020.2 | loss tokens/s: 59486.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.524359E-01 | lm-global-supervised-tokens: 1.319282E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.202398E-02 | lm-token-buffer-mib: 2.325682E+02 | global_load_balancing_loss: 9.988551E-01 | loss scale: 1.0 | grad norm: 0.230 | params norm: 11187.186 | params rms: 0.060575 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:18:29.255599 | finish at 2026-10-17 17:59:26 + [2026-10-14 03:44:44] iteration 984/ 2384 | consumed samples: 31488 | elapsed time per iteration (ms): 228180.7 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.04% | FLOPs estimator: workload-aware | logical tokens/s: 73525.9 | valid tokens/s: 73525.9 | physical tokens/s: 73968.9 | loss tokens/s: 55426.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.411375E-01 | lm-global-supervised-tokens: 1.264734E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.957076E-02 | lm-token-buffer-mib: 2.330129E+02 | global_load_balancing_loss: 1.000577E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11187.146 | params rms: 0.060575 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:44:12.967405 | finish at 2026-10-17 20:28:57 + [2026-10-14 03:48:26] iteration 985/ 2384 | consumed samples: 31520 | elapsed time per iteration (ms): 221288.1 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75816.0 | valid tokens/s: 75816.0 | physical tokens/s: 76515.5 | loss tokens/s: 59008.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.187114E-01 | lm-global-supervised-tokens: 1.305780E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.632472E-02 | lm-token-buffer-mib: 2.350279E+02 | global_load_balancing_loss: 9.998204E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11187.113 | params rms: 0.060575 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:59:42.121284 | finish at 2026-10-17 17:48:08 + [2026-10-14 03:52:06] iteration 986/ 2384 | consumed samples: 31552 | elapsed time per iteration (ms): 220534.5 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 76075.2 | valid tokens/s: 76075.2 | physical tokens/s: 76642.6 | loss tokens/s: 56433.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.668460E-01 | lm-global-supervised-tokens: 1.244560E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.661248E-02 | lm-token-buffer-mib: 2.329836E+02 | global_load_balancing_loss: 1.001741E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11187.082 | params rms: 0.060574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:38:27.166175 | finish at 2026-10-17 17:30:33 + [2026-10-14 03:55:50] iteration 987/ 2384 | consumed samples: 31584 | elapsed time per iteration (ms): 223797.9 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 74965.8 | valid tokens/s: 74965.8 | physical tokens/s: 76457.3 | loss tokens/s: 58758.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.339434E-01 | lm-global-supervised-tokens: 1.315002E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.391366E-02 | lm-token-buffer-mib: 2.355314E+02 | global_load_balancing_loss: 1.000952E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11187.052 | params rms: 0.060574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:50:45.714875 | finish at 2026-10-17 18:46:36 + [2026-10-14 03:59:33] iteration 988/ 2384 | consumed samples: 31616 | elapsed time per iteration (ms): 223326.7 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.27% | FLOPs estimator: workload-aware | logical tokens/s: 75124.0 | valid tokens/s: 75124.0 | physical tokens/s: 75997.3 | loss tokens/s: 55361.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.946305E-01 | lm-global-supervised-tokens: 1.236359E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.812280E-02 | lm-token-buffer-mib: 2.340161E+02 | global_load_balancing_loss: 1.001910E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11187.025 | params rms: 0.060574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:36:04.009923 | finish at 2026-10-17 18:35:37 + [2026-10-14 04:03:15] iteration 989/ 2384 | consumed samples: 31648 | elapsed time per iteration (ms): 221497.9 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75744.3 | valid tokens/s: 75744.3 | physical tokens/s: 76540.7 | loss tokens/s: 60626.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.990389E-01 | lm-global-supervised-tokens: 1.342866E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.208784E-02 | lm-token-buffer-mib: 2.342412E+02 | global_load_balancing_loss: 9.997748E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11186.998 | params rms: 0.060574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:49:49.615080 | finish at 2026-10-17 17:53:05 + [2026-10-14 04:07:18] iteration 990/ 2384 | consumed samples: 31680 | elapsed time per iteration (ms): 243475.6 | throughput per GPU (TFLOP/s/GPU): 18.4 | MFU 5.91% | FLOPs estimator: workload-aware | logical tokens/s: 68907.1 | valid tokens/s: 68907.1 | physical tokens/s: 69652.6 | loss tokens/s: 53353.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.568887E-01 | lm-global-supervised-tokens: 1.299038E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.011655E-02 | lm-token-buffer-mib: 2.333877E+02 | global_load_balancing_loss: 9.996936E-01 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11186.968 | params rms: 0.060574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:16:45.044236 | finish at 2026-10-18 02:24:03 + [2026-10-14 04:10:59] iteration 991/ 2384 | consumed samples: 31712 | elapsed time per iteration (ms): 220215.1 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 76185.6 | valid tokens/s: 76185.6 | physical tokens/s: 76903.4 | loss tokens/s: 58041.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.236853E-01 | lm-global-supervised-tokens: 1.278155E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.339634E-02 | lm-token-buffer-mib: 2.333896E+02 | global_load_balancing_loss: 1.000455E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11186.938 | params rms: 0.060574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:12:39.617100 | finish at 2026-10-17 17:23:38 + [2026-10-14 04:14:40] iteration 992/ 2384 | consumed samples: 31744 | elapsed time per iteration (ms): 221729.2 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.44% | FLOPs estimator: workload-aware | logical tokens/s: 75665.1 | valid tokens/s: 75665.1 | physical tokens/s: 76176.5 | loss tokens/s: 55005.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.497244E-01 | lm-global-supervised-tokens: 1.219642E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.799822E-02 | lm-token-buffer-mib: 2.334988E+02 | global_load_balancing_loss: 9.986709E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11186.910 | params rms: 0.060573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:44:06.994137 | finish at 2026-10-17 17:58:47 + [2026-10-14 04:18:19] iteration 993/ 2384 | consumed samples: 31776 | elapsed time per iteration (ms): 218604.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 76746.8 | valid tokens/s: 76746.8 | physical tokens/s: 76928.0 | loss tokens/s: 53190.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.430862E-01 | lm-global-supervised-tokens: 1.162773E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.247158E-02 | lm-token-buffer-mib: 2.322444E+02 | global_load_balancing_loss: 9.989948E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11186.881 | params rms: 0.060573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:27:58.335254 | finish at 2026-10-17 16:46:17 + [2026-10-14 04:22:02] iteration 994/ 2384 | consumed samples: 31808 | elapsed time per iteration (ms): 222820.8 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75294.6 | valid tokens/s: 75294.6 | physical tokens/s: 76038.0 | loss tokens/s: 57466.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.635076E-01 | lm-global-supervised-tokens: 1.280481E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.504499E-02 | lm-token-buffer-mib: 2.355837E+02 | global_load_balancing_loss: 1.002524E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11186.849 | params rms: 0.060573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:02:00.861387 | finish at 2026-10-17 18:24:03 + [2026-10-14 04:25:45] iteration 995/ 2384 | consumed samples: 31840 | elapsed time per iteration (ms): 223578.3 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75039.4 | valid tokens/s: 75039.4 | physical tokens/s: 75761.8 | loss tokens/s: 51810.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.210961E-01 | lm-global-supervised-tokens: 1.158369E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.053175E-02 | lm-token-buffer-mib: 2.342620E+02 | global_load_balancing_loss: 1.001479E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11186.815 | params rms: 0.060573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:15:50.216310 | finish at 2026-10-17 18:41:36 + [2026-10-14 04:29:31] iteration 996/ 2384 | consumed samples: 31872 | elapsed time per iteration (ms): 225303.6 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 74464.7 | valid tokens/s: 74464.7 | physical tokens/s: 75115.7 | loss tokens/s: 58541.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.756017E-01 | lm-global-supervised-tokens: 1.318955E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.450563E-02 | lm-token-buffer-mib: 2.333459E+02 | global_load_balancing_loss: 1.001698E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11186.783 | params rms: 0.060573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:52:01.419735 | finish at 2026-10-17 19:21:32 + [2026-10-14 04:33:14] iteration 997/ 2384 | consumed samples: 31904 | elapsed time per iteration (ms): 223356.7 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.70% | FLOPs estimator: workload-aware | logical tokens/s: 75113.7 | valid tokens/s: 75113.7 | physical tokens/s: 75449.5 | loss tokens/s: 56514.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.213450E-01 | lm-global-supervised-tokens: 1.262278E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.251522E-02 | lm-token-buffer-mib: 2.325446E+02 | global_load_balancing_loss: 1.000000E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11186.753 | params rms: 0.060573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:03:15.789448 | finish at 2026-10-17 18:36:30 + [2026-10-14 04:36:56] iteration 998/ 2384 | consumed samples: 31936 | elapsed time per iteration (ms): 222103.9 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 75537.6 | valid tokens/s: 75537.6 | physical tokens/s: 76020.6 | loss tokens/s: 57678.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.432881E-01 | lm-global-supervised-tokens: 1.281064E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.585545E-02 | lm-token-buffer-mib: 2.329238E+02 | global_load_balancing_loss: 1.000805E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11186.722 | params rms: 0.060572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:30:35.979233 | finish at 2026-10-17 18:07:32 + [2026-10-14 04:40:42] iteration 999/ 2384 | consumed samples: 31968 | elapsed time per iteration (ms): 225476.3 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 74407.7 | valid tokens/s: 74407.7 | physical tokens/s: 75176.2 | loss tokens/s: 57757.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.520673E-01 | lm-global-supervised-tokens: 1.302294E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.508743E-02 | lm-token-buffer-mib: 2.351771E+02 | global_load_balancing_loss: 9.992158E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11186.694 | params rms: 0.060572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:44:44.704560 | finish at 2026-10-17 19:25:26 + [2026-10-14 04:44:43] iteration 1000/ 2384 | consumed samples: 32000 | elapsed time per iteration (ms): 241523.7 | throughput per GPU (TFLOP/s/GPU): 17.1 | MFU 5.48% | FLOPs estimator: workload-aware | logical tokens/s: 69464.0 | valid tokens/s: 69464.0 | physical tokens/s: 70043.6 | loss tokens/s: 52571.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.863227E-01 | lm-global-supervised-tokens: 1.269728E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.250993E-02 | lm-token-buffer-mib: 2.341956E+02 | global_load_balancing_loss: 1.000100E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11186.664 | params rms: 0.060572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:51:08.865143 | finish at 2026-10-18 01:35:52 +times across ranks (ms): + save-checkpoint: + rank 0: 24809.33 + rank 1: 24809.36 + rank 2: 24809.36 + rank 3: 24809.35 + rank 4: 24809.34 + rank 5: 24809.37 + rank 6: 24809.37 + rank 7: 24809.34 + rank 8: 24809.11 + rank 9: 24809.14 + rank 10: 24809.10 + rank 11: 24809.10 + rank 12: 24809.12 + rank 13: 24809.12 + rank 14: 24809.11 + rank 15: 24809.15 + rank 16: 24810.18 + rank 17: 24810.25 + rank 18: 24810.19 + rank 19: 24810.17 + rank 20: 24810.13 + rank 21: 24810.15 + rank 22: 24810.13 + rank 23: 24810.12 + rank 24: 24809.35 + rank 25: 24809.40 + rank 26: 24809.48 + rank 27: 24809.82 + rank 28: 24809.34 + rank 29: 24809.38 + rank 30: 24809.32 + rank 31: 24809.36 + rank 32: 24809.21 + rank 33: 24809.26 + rank 34: 24809.31 + rank 35: 24809.24 + rank 36: 24809.19 + rank 37: 24809.17 + rank 38: 24809.34 + rank 39: 24809.12 + rank 40: 24809.09 + rank 41: 24809.09 + rank 42: 24809.07 + rank 43: 24808.93 + rank 44: 24809.10 + rank 45: 24809.05 + rank 46: 24809.05 + rank 47: 24809.04 + rank 48: 24808.49 + rank 49: 24808.57 + rank 50: 24808.51 + rank 51: 24808.48 + rank 52: 24808.54 + rank 53: 24808.61 + rank 54: 24808.57 + rank 55: 24808.52 + rank 56: 24809.17 + rank 57: 24809.20 + rank 58: 24809.26 + rank 59: 24809.19 + rank 60: 24809.21 + rank 61: 24809.16 + rank 62: 24809.18 + rank 63: 24809.17 + [2026-10-14 04:49:14] iteration 1001/ 2384 | consumed samples: 32032 | elapsed time per iteration (ms): 245913.9 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 68223.7 | valid tokens/s: 68223.7 | physical tokens/s: 69085.9 | loss tokens/s: 53478.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.105592E-01 | lm-global-supervised-tokens: 1.315110E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 7.470369E-02 | lm-token-buffer-mib: 2.355911E+02 | global_load_balancing_loss: 1.000921E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11186.635 | params rms: 0.060572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:28:18.883332 | finish at 2026-10-18 03:17:33 +times across ranks (ms): + forward-backward: + rank 0: 245088.81 + rank 1: 245090.97 + rank 2: 245090.23 + rank 3: 245094.62 + rank 4: 245095.08 + rank 5: 245096.28 + rank 6: 245089.52 + rank 7: 245091.20 + rank 8: 245636.19 + rank 9: 245647.88 + rank 10: 245643.39 + rank 11: 245644.42 + rank 12: 245648.38 + rank 13: 245648.03 + rank 14: 245636.81 + rank 15: 245644.41 + rank 16: 245629.42 + rank 17: 245634.48 + rank 18: 245633.52 + rank 19: 245636.78 + rank 20: 245640.61 + rank 21: 245640.89 + rank 22: 245635.42 + rank 23: 245630.80 + rank 24: 245633.84 + rank 25: 245644.59 + rank 26: 245642.81 + rank 27: 245640.61 + rank 28: 245643.58 + rank 29: 245639.89 + rank 30: 245641.02 + rank 31: 245636.69 + rank 32: 245074.53 + rank 33: 245075.23 + rank 34: 245073.02 + rank 35: 245076.09 + rank 36: 245076.36 + rank 37: 245077.42 + rank 38: 245074.36 + rank 39: 245075.56 + rank 40: 245628.39 + rank 41: 245638.88 + rank 42: 245633.80 + rank 43: 245633.11 + rank 44: 245636.14 + rank 45: 245635.50 + rank 46: 245631.88 + rank 47: 245638.30 + rank 48: 245625.41 + rank 49: 245628.89 + rank 50: 245628.11 + rank 51: 245630.06 + rank 52: 245633.41 + rank 53: 245632.69 + rank 54: 245633.80 + rank 55: 245627.62 + rank 56: 245645.97 + rank 57: 245652.23 + rank 58: 245651.98 + rank 59: 245649.11 + rank 60: 245649.77 + rank 61: 245645.31 + rank 62: 245653.45 + rank 63: 245648.86 + forward-compute: + rank 0: 146876.77 + rank 1: 146879.17 + rank 2: 146874.33 + rank 3: 146875.45 + rank 4: 146869.38 + rank 5: 146878.28 + rank 6: 146879.22 + rank 7: 146881.17 + rank 8: 147512.06 + rank 9: 147524.08 + rank 10: 147523.00 + rank 11: 147517.88 + rank 12: 147521.17 + rank 13: 147521.50 + rank 14: 147518.17 + rank 15: 147526.20 + rank 16: 147413.48 + rank 17: 147417.59 + rank 18: 147416.97 + rank 19: 147418.23 + rank 20: 147419.09 + rank 21: 147417.69 + rank 22: 147418.41 + rank 23: 147420.62 + rank 24: 147430.02 + rank 25: 147436.52 + rank 26: 147434.97 + rank 27: 147433.64 + rank 28: 147431.72 + rank 29: 147427.28 + rank 30: 147433.38 + rank 31: 147427.78 + rank 32: 59027.99 + rank 33: 59048.03 + rank 34: 59054.38 + rank 35: 59064.59 + rank 36: 59064.62 + rank 37: 59078.88 + rank 38: 59060.59 + rank 39: 59061.25 + rank 40: 59650.97 + rank 41: 59637.23 + rank 42: 59630.93 + rank 43: 59636.82 + rank 44: 59623.75 + rank 45: 59652.07 + rank 46: 59603.73 + rank 47: 59643.73 + rank 48: 59597.16 + rank 49: 59597.99 + rank 50: 59599.89 + rank 51: 59589.55 + rank 52: 59650.96 + rank 53: 59653.60 + rank 54: 59594.23 + rank 55: 59606.91 + rank 56: 59613.82 + rank 57: 59617.97 + rank 58: 59600.69 + rank 59: 59605.55 + rank 60: 59615.42 + rank 61: 59621.52 + rank 62: 59601.17 + rank 63: 59654.89 + backward-compute: + rank 0: 98159.09 + rank 1: 98154.92 + rank 2: 98163.74 + rank 3: 98165.16 + rank 4: 98165.29 + rank 5: 98166.77 + rank 6: 98159.98 + rank 7: 98156.45 + rank 8: 98070.36 + rank 9: 98070.18 + rank 10: 98065.95 + rank 11: 98071.47 + rank 12: 98070.33 + rank 13: 98071.48 + rank 14: 98064.14 + rank 15: 98063.28 + rank 16: 98169.50 + rank 17: 98160.91 + rank 18: 98160.03 + rank 19: 98163.34 + rank 20: 98167.55 + rank 21: 98168.22 + rank 22: 98164.53 + rank 23: 98158.65 + rank 24: 98145.09 + rank 25: 98143.65 + rank 26: 98144.03 + rank 27: 98142.79 + rank 28: 98143.62 + rank 29: 98150.03 + rank 30: 98145.93 + rank 31: 98137.26 + rank 32: 186006.14 + rank 33: 185987.55 + rank 34: 185979.72 + rank 35: 185973.14 + rank 36: 185974.14 + rank 37: 185960.92 + rank 38: 185974.70 + rank 39: 185977.00 + rank 40: 185942.22 + rank 41: 185967.78 + rank 42: 185969.62 + rank 43: 185961.88 + rank 44: 185978.47 + rank 45: 185948.81 + rank 46: 185992.23 + rank 47: 185959.05 + rank 48: 185990.00 + rank 49: 185992.25 + rank 50: 185984.98 + rank 51: 186003.53 + rank 52: 185942.14 + rank 53: 185940.39 + rank 54: 185998.91 + rank 55: 185982.70 + rank 56: 185999.33 + rank 57: 185999.06 + rank 58: 186015.41 + rank 59: 186009.19 + rank 60: 185999.62 + rank 61: 185986.30 + rank 62: 186018.02 + rank 63: 185959.48 + batch-generator: + rank 0: 36.86 + rank 1: 35.94 + rank 2: 37.84 + rank 3: 35.96 + rank 4: 38.52 + rank 5: 37.83 + rank 6: 37.51 + rank 7: 36.71 + rank 8: 69.31 + rank 9: 70.46 + rank 10: 70.08 + rank 11: 69.82 + rank 12: 70.68 + rank 13: 71.15 + rank 14: 71.58 + rank 15: 71.09 + rank 16: 33.97 + rank 17: 37.54 + rank 18: 37.59 + rank 19: 30.21 + rank 20: 36.50 + rank 21: 37.28 + rank 22: 37.81 + rank 23: 35.99 + rank 24: 39.00 + rank 25: 45.84 + rank 26: 50.75 + rank 27: 50.91 + rank 28: 51.53 + rank 29: 49.59 + rank 30: 46.41 + rank 31: 46.27 + rank 32: 48.71 + rank 33: 51.94 + rank 34: 48.85 + rank 35: 42.42 + rank 36: 41.07 + rank 37: 42.45 + rank 38: 45.99 + rank 39: 41.78 + rank 40: 37.33 + rank 41: 36.63 + rank 42: 35.17 + rank 43: 37.95 + rank 44: 36.23 + rank 45: 35.85 + rank 46: 38.53 + rank 47: 32.65 + rank 48: 56.03 + rank 49: 55.29 + rank 50: 62.33 + rank 51: 49.49 + rank 52: 53.03 + rank 53: 52.92 + rank 54: 39.23 + rank 55: 52.78 + rank 56: 35.19 + rank 57: 38.35 + rank 58: 38.70 + rank 59: 38.20 + rank 60: 37.72 + rank 61: 37.18 + rank 62: 27.93 + rank 63: 33.63 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.04 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.03 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.02 + rank 48: 0.04 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.07 + rank 1: 0.85 + rank 2: 0.90 + rank 3: 1.28 + rank 4: 0.90 + rank 5: 0.84 + rank 6: 0.93 + rank 7: 0.80 + rank 8: 1.31 + rank 9: 1.05 + rank 10: 0.85 + rank 11: 1.25 + rank 12: 1.21 + rank 13: 0.83 + rank 14: 0.86 + rank 15: 1.18 + rank 16: 1.43 + rank 17: 0.80 + rank 18: 0.99 + rank 19: 0.91 + rank 20: 0.86 + rank 21: 1.24 + rank 22: 0.83 + rank 23: 1.16 + rank 24: 1.18 + rank 25: 1.48 + rank 26: 1.47 + rank 27: 1.39 + rank 28: 1.74 + rank 29: 1.81 + rank 30: 1.68 + rank 31: 1.33 + rank 32: 1.25 + rank 33: 1.31 + rank 34: 1.50 + rank 35: 1.34 + rank 36: 1.27 + rank 37: 1.29 + rank 38: 1.24 + rank 39: 1.29 + rank 40: 0.84 + rank 41: 0.82 + rank 42: 0.78 + rank 43: 0.79 + rank 44: 0.86 + rank 45: 0.88 + rank 46: 1.20 + rank 47: 0.86 + rank 48: 1.09 + rank 49: 1.02 + rank 50: 1.07 + rank 51: 0.81 + rank 52: 1.05 + rank 53: 0.86 + rank 54: 0.94 + rank 55: 1.28 + rank 56: 1.02 + rank 57: 0.92 + rank 58: 1.13 + rank 59: 0.83 + rank 60: 1.05 + rank 61: 0.94 + rank 62: 0.83 + rank 63: 0.83 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.03 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.02 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.27 + rank 1: 1.88 + rank 2: 1.82 + rank 3: 2.26 + rank 4: 1.77 + rank 5: 1.93 + rank 6: 2.02 + rank 7: 1.95 + rank 8: 1.85 + rank 9: 2.20 + rank 10: 1.80 + rank 11: 1.76 + rank 12: 1.99 + rank 13: 1.82 + rank 14: 1.96 + rank 15: 2.25 + rank 16: 2.58 + rank 17: 2.38 + rank 18: 2.32 + rank 19: 2.29 + rank 20: 2.29 + rank 21: 2.41 + rank 22: 2.68 + rank 23: 2.49 + rank 24: 2.00 + rank 25: 2.35 + rank 26: 1.98 + rank 27: 2.18 + rank 28: 2.54 + rank 29: 2.25 + rank 30: 2.17 + rank 31: 2.26 + rank 32: 2.11 + rank 33: 2.03 + rank 34: 2.09 + rank 35: 2.33 + rank 36: 2.15 + rank 37: 2.26 + rank 38: 2.32 + rank 39: 1.96 + rank 40: 1.83 + rank 41: 1.89 + rank 42: 1.97 + rank 43: 1.94 + rank 44: 1.88 + rank 45: 2.12 + rank 46: 1.77 + rank 47: 1.79 + rank 48: 1.93 + rank 49: 1.80 + rank 50: 2.48 + rank 51: 1.73 + rank 52: 2.03 + rank 53: 1.76 + rank 54: 1.80 + rank 55: 1.82 + rank 56: 1.81 + rank 57: 2.50 + rank 58: 1.82 + rank 59: 1.76 + rank 60: 1.81 + rank 61: 1.76 + rank 62: 1.83 + rank 63: 1.76 + optimizer-inner-step: + rank 0: 51.19 + rank 1: 51.18 + rank 2: 50.94 + rank 3: 51.10 + rank 4: 50.87 + rank 5: 50.97 + rank 6: 51.35 + rank 7: 51.32 + rank 8: 51.14 + rank 9: 51.14 + rank 10: 50.97 + rank 11: 51.13 + rank 12: 51.15 + rank 13: 51.39 + rank 14: 51.10 + rank 15: 50.94 + rank 16: 51.50 + rank 17: 51.17 + rank 18: 51.28 + rank 19: 51.22 + rank 20: 51.12 + rank 21: 51.08 + rank 22: 51.71 + rank 23: 51.18 + rank 24: 50.50 + rank 25: 51.04 + rank 26: 50.99 + rank 27: 50.72 + rank 28: 51.44 + rank 29: 50.80 + rank 30: 50.97 + rank 31: 51.03 + rank 32: 51.19 + rank 33: 51.07 + rank 34: 51.12 + rank 35: 51.47 + rank 36: 51.27 + rank 37: 51.10 + rank 38: 51.02 + rank 39: 51.02 + rank 40: 51.09 + rank 41: 51.21 + rank 42: 51.22 + rank 43: 51.31 + rank 44: 50.90 + rank 45: 51.56 + rank 46: 51.14 + rank 47: 50.85 + rank 48: 48.48 + rank 49: 48.41 + rank 50: 60.25 + rank 51: 48.24 + rank 52: 48.52 + rank 53: 48.39 + rank 54: 48.59 + rank 55: 48.48 + rank 56: 50.72 + rank 57: 51.93 + rank 58: 51.01 + rank 59: 50.90 + rank 60: 51.24 + rank 61: 50.91 + rank 62: 50.95 + rank 63: 50.88 + optimizer-copy-main-to-model-params: + rank 0: 4.38 + rank 1: 3.82 + rank 2: 3.85 + rank 3: 4.42 + rank 4: 3.72 + rank 5: 4.01 + rank 6: 4.09 + rank 7: 4.04 + rank 8: 3.89 + rank 9: 4.42 + rank 10: 3.86 + rank 11: 3.84 + rank 12: 3.97 + rank 13: 3.87 + rank 14: 4.09 + rank 15: 4.26 + rank 16: 4.43 + rank 17: 4.06 + rank 18: 4.43 + rank 19: 4.02 + rank 20: 4.18 + rank 21: 4.34 + rank 22: 4.79 + rank 23: 4.33 + rank 24: 4.02 + rank 25: 4.63 + rank 26: 4.00 + rank 27: 4.14 + rank 28: 4.49 + rank 29: 4.43 + rank 30: 4.37 + rank 31: 4.14 + rank 32: 4.16 + rank 33: 3.96 + rank 34: 4.01 + rank 35: 4.59 + rank 36: 4.26 + rank 37: 4.17 + rank 38: 4.49 + rank 39: 4.21 + rank 40: 3.83 + rank 41: 3.97 + rank 42: 3.95 + rank 43: 4.05 + rank 44: 3.96 + rank 45: 4.29 + rank 46: 3.74 + rank 47: 3.73 + rank 48: 4.05 + rank 49: 3.92 + rank 50: 5.03 + rank 51: 3.96 + rank 52: 4.19 + rank 53: 3.91 + rank 54: 3.94 + rank 55: 3.92 + rank 56: 3.73 + rank 57: 4.79 + rank 58: 3.94 + rank 59: 3.74 + rank 60: 3.74 + rank 61: 3.69 + rank 62: 3.89 + rank 63: 3.68 + optimizer: + rank 0: 66.31 + rank 1: 65.43 + rank 2: 65.16 + rank 3: 66.13 + rank 4: 64.96 + rank 5: 65.48 + rank 6: 65.96 + rank 7: 65.88 + rank 8: 65.50 + rank 9: 66.22 + rank 10: 65.26 + rank 11: 65.38 + rank 12: 65.60 + rank 13: 65.64 + rank 14: 65.65 + rank 15: 65.78 + rank 16: 66.79 + rank 17: 65.98 + rank 18: 66.44 + rank 19: 65.92 + rank 20: 66.02 + rank 21: 66.23 + rank 22: 67.43 + rank 23: 66.47 + rank 24: 65.31 + rank 25: 66.37 + rank 26: 65.56 + rank 27: 65.81 + rank 28: 66.59 + rank 29: 65.97 + rank 30: 66.01 + rank 31: 65.86 + rank 32: 65.80 + rank 33: 65.38 + rank 34: 65.52 + rank 35: 66.71 + rank 36: 66.05 + rank 37: 65.72 + rank 38: 66.10 + rank 39: 65.65 + rank 40: 65.28 + rank 41: 65.64 + rank 42: 65.62 + rank 43: 65.83 + rank 44: 65.34 + rank 45: 66.41 + rank 46: 65.25 + rank 47: 64.97 + rank 48: 65.67 + rank 49: 65.36 + rank 50: 78.72 + rank 51: 65.22 + rank 52: 65.91 + rank 53: 65.34 + rank 54: 65.56 + rank 55: 65.43 + rank 56: 64.90 + rank 57: 67.60 + rank 58: 65.44 + rank 59: 65.01 + rank 60: 65.38 + rank 61: 65.02 + rank 62: 65.30 + rank 63: 64.95 + interval-time: + rank 0: 0.97 + rank 1: 0.94 + rank 2: 0.95 + rank 3: 1.84 + rank 4: 0.89 + rank 5: 1.00 + rank 6: 0.96 + rank 7: 1.01 + rank 8: 0.83 + rank 9: 0.85 + rank 10: 0.86 + rank 11: 0.83 + rank 12: 0.84 + rank 13: 0.73 + rank 14: 0.85 + rank 15: 0.85 + rank 16: 0.86 + rank 17: 0.85 + rank 18: 0.87 + rank 19: 0.88 + rank 20: 0.87 + rank 21: 0.86 + rank 22: 0.77 + rank 23: 0.77 + rank 24: 0.97 + rank 25: 0.98 + rank 26: 0.86 + rank 27: 0.95 + rank 28: 0.95 + rank 29: 0.94 + rank 30: 0.89 + rank 31: 1.09 + rank 32: 0.96 + rank 33: 0.98 + rank 34: 0.96 + rank 35: 0.86 + rank 36: 0.81 + rank 37: 0.85 + rank 38: 0.82 + rank 39: 0.84 + rank 40: 0.85 + rank 41: 0.84 + rank 42: 0.82 + rank 43: 0.76 + rank 44: 0.74 + rank 45: 0.83 + rank 46: 0.83 + rank 47: 0.82 + rank 48: 9.21 + rank 49: 9.19 + rank 50: 9.20 + rank 51: 9.18 + rank 52: 9.20 + rank 53: 9.20 + rank 54: 9.21 + rank 55: 9.18 + rank 56: 0.97 + rank 57: 0.89 + rank 58: 0.91 + rank 59: 0.87 + rank 60: 0.92 + rank 61: 0.93 + rank 62: 0.94 + rank 63: 1.46 + [2026-10-14 04:53:03] iteration 1002/ 2384 | consumed samples: 32064 | elapsed time per iteration (ms): 229610.4 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 73068.1 | valid tokens/s: 73068.1 | physical tokens/s: 73773.7 | loss tokens/s: 52419.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.524406E-01 | lm-global-supervised-tokens: 1.203612E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.713766E-02 | lm-token-buffer-mib: 2.340956E+02 | global_load_balancing_loss: 9.993061E-01 | loss scale: 1.0 | grad norm: 0.251 | params norm: 11186.606 | params rms: 0.060572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:08:41.518380 | finish at 2026-10-17 21:01:45 +times across ranks (ms): + forward-backward: + rank 0: 223245.02 + rank 1: 223326.61 + rank 2: 223321.97 + rank 3: 223324.11 + rank 4: 223326.61 + rank 5: 223322.83 + rank 6: 223325.45 + rank 7: 223324.05 + rank 8: 223326.08 + rank 9: 223324.72 + rank 10: 223325.05 + rank 11: 223323.31 + rank 12: 223324.69 + rank 13: 223326.72 + rank 14: 223328.00 + rank 15: 223326.91 + rank 16: 223327.59 + rank 17: 223327.56 + rank 18: 223328.34 + rank 19: 223327.92 + rank 20: 223325.23 + rank 21: 223327.16 + rank 22: 223324.61 + rank 23: 223326.16 + rank 24: 223326.17 + rank 25: 223318.86 + rank 26: 223315.31 + rank 27: 223325.30 + rank 28: 223322.30 + rank 29: 223325.44 + rank 30: 223320.61 + rank 31: 223326.95 + rank 32: 223246.12 + rank 33: 223329.61 + rank 34: 223322.86 + rank 35: 223323.42 + rank 36: 223325.62 + rank 37: 223322.77 + rank 38: 223324.25 + rank 39: 223327.89 + rank 40: 223328.02 + rank 41: 223327.50 + rank 42: 223326.12 + rank 43: 223324.98 + rank 44: 223326.14 + rank 45: 223326.27 + rank 46: 223326.28 + rank 47: 223330.36 + rank 48: 223327.02 + rank 49: 223326.41 + rank 50: 223325.45 + rank 51: 223325.30 + rank 52: 223322.02 + rank 53: 223323.97 + rank 54: 223324.12 + rank 55: 223322.59 + rank 56: 223328.27 + rank 57: 223322.70 + rank 58: 223319.64 + rank 59: 223329.89 + rank 60: 223326.19 + rank 61: 223329.70 + rank 62: 223326.00 + rank 63: 223328.50 + forward-compute: + rank 0: 114469.48 + rank 1: 114547.39 + rank 2: 114537.41 + rank 3: 114529.35 + rank 4: 114513.04 + rank 5: 114510.02 + rank 6: 114456.94 + rank 7: 114540.48 + rank 8: 114555.74 + rank 9: 114558.68 + rank 10: 114560.91 + rank 11: 114549.68 + rank 12: 114559.51 + rank 13: 114563.79 + rank 14: 114555.02 + rank 15: 114565.85 + rank 16: 114536.91 + rank 17: 114526.16 + rank 18: 114512.90 + rank 19: 114508.60 + rank 20: 114503.02 + rank 21: 114505.57 + rank 22: 114501.65 + rank 23: 114503.42 + rank 24: 114512.36 + rank 25: 114505.42 + rank 26: 114499.51 + rank 27: 114505.58 + rank 28: 114505.05 + rank 29: 114509.40 + rank 30: 114496.32 + rank 31: 114496.20 + rank 32: 62972.00 + rank 33: 63067.85 + rank 34: 63051.09 + rank 35: 63066.29 + rank 36: 63081.55 + rank 37: 63092.05 + rank 38: 63064.27 + rank 39: 63069.20 + rank 40: 63096.31 + rank 41: 63058.11 + rank 42: 63078.89 + rank 43: 63076.16 + rank 44: 63073.60 + rank 45: 63078.25 + rank 46: 63055.07 + rank 47: 63085.56 + rank 48: 63065.04 + rank 49: 63067.31 + rank 50: 63071.41 + rank 51: 63067.24 + rank 52: 63085.92 + rank 53: 63076.70 + rank 54: 63019.71 + rank 55: 63052.31 + rank 56: 63061.11 + rank 57: 63044.35 + rank 58: 63044.84 + rank 59: 63044.70 + rank 60: 63044.55 + rank 61: 63064.52 + rank 62: 63026.84 + rank 63: 63082.06 + backward-compute: + rank 0: 108728.05 + rank 1: 108728.12 + rank 2: 108726.09 + rank 3: 108743.28 + rank 4: 108757.08 + rank 5: 108759.48 + rank 6: 108816.65 + rank 7: 108732.19 + rank 8: 108716.84 + rank 9: 108713.51 + rank 10: 108713.28 + rank 11: 108723.77 + rank 12: 108714.47 + rank 13: 108712.59 + rank 14: 108722.01 + rank 15: 108710.87 + rank 16: 108743.98 + rank 17: 108746.23 + rank 18: 108766.47 + rank 19: 108769.62 + rank 20: 108773.18 + rank 21: 108771.36 + rank 22: 108773.01 + rank 23: 108778.01 + rank 24: 108757.66 + rank 25: 108755.34 + rank 26: 108754.24 + rank 27: 108757.18 + rank 28: 108755.23 + rank 29: 108758.93 + rank 30: 108766.03 + rank 31: 108769.69 + rank 32: 160235.38 + rank 33: 160220.98 + rank 34: 160233.31 + rank 35: 160215.94 + rank 36: 160206.34 + rank 37: 160189.52 + rank 38: 160221.06 + rank 39: 160220.67 + rank 40: 160197.55 + rank 41: 160234.02 + rank 42: 160214.14 + rank 43: 160213.67 + rank 44: 160217.25 + rank 45: 160213.67 + rank 46: 160234.44 + rank 47: 160210.72 + rank 48: 160227.67 + rank 49: 160223.66 + rank 50: 160217.36 + rank 51: 160228.09 + rank 52: 160201.80 + rank 53: 160213.58 + rank 54: 160268.12 + rank 55: 160237.80 + rank 56: 160235.00 + rank 57: 160242.56 + rank 58: 160238.23 + rank 59: 160250.31 + rank 60: 160246.20 + rank 61: 160230.28 + rank 62: 160265.55 + rank 63: 160212.27 + batch-generator: + rank 0: 36.48 + rank 1: 36.99 + rank 2: 36.45 + rank 3: 37.67 + rank 4: 37.51 + rank 5: 38.90 + rank 6: 37.74 + rank 7: 35.86 + rank 8: 55.10 + rank 9: 55.87 + rank 10: 56.96 + rank 11: 55.80 + rank 12: 58.67 + rank 13: 56.23 + rank 14: 54.93 + rank 15: 56.38 + rank 16: 36.48 + rank 17: 37.13 + rank 18: 38.24 + rank 19: 28.73 + rank 20: 37.07 + rank 21: 39.27 + rank 22: 40.16 + rank 23: 36.70 + rank 24: 37.49 + rank 25: 48.01 + rank 26: 51.63 + rank 27: 49.96 + rank 28: 47.93 + rank 29: 50.07 + rank 30: 44.84 + rank 31: 47.97 + rank 32: 44.73 + rank 33: 50.97 + rank 34: 46.86 + rank 35: 40.34 + rank 36: 41.20 + rank 37: 43.09 + rank 38: 43.02 + rank 39: 40.91 + rank 40: 36.44 + rank 41: 37.14 + rank 42: 35.87 + rank 43: 36.61 + rank 44: 33.81 + rank 45: 35.51 + rank 46: 36.60 + rank 47: 32.15 + rank 48: 39.52 + rank 49: 45.70 + rank 50: 45.82 + rank 51: 34.16 + rank 52: 40.89 + rank 53: 41.11 + rank 54: 37.46 + rank 55: 34.35 + rank 56: 34.56 + rank 57: 39.02 + rank 58: 39.86 + rank 59: 37.92 + rank 60: 37.74 + rank 61: 39.90 + rank 62: 28.02 + rank 63: 35.40 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.04 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.04 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.04 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.31 + rank 1: 0.90 + rank 2: 0.81 + rank 3: 1.36 + rank 4: 0.96 + rank 5: 1.15 + rank 6: 0.87 + rank 7: 0.84 + rank 8: 1.18 + rank 9: 1.15 + rank 10: 1.00 + rank 11: 0.81 + rank 12: 0.90 + rank 13: 0.83 + rank 14: 0.96 + rank 15: 0.83 + rank 16: 1.07 + rank 17: 0.86 + rank 18: 1.03 + rank 19: 0.96 + rank 20: 0.91 + rank 21: 1.18 + rank 22: 0.86 + rank 23: 0.83 + rank 24: 1.33 + rank 25: 1.29 + rank 26: 1.67 + rank 27: 1.50 + rank 28: 1.43 + rank 29: 1.47 + rank 30: 1.66 + rank 31: 1.35 + rank 32: 1.62 + rank 33: 1.72 + rank 34: 1.33 + rank 35: 1.50 + rank 36: 1.35 + rank 37: 1.34 + rank 38: 1.46 + rank 39: 1.19 + rank 40: 0.94 + rank 41: 0.95 + rank 42: 0.88 + rank 43: 0.94 + rank 44: 1.55 + rank 45: 0.79 + rank 46: 0.85 + rank 47: 1.50 + rank 48: 1.27 + rank 49: 1.00 + rank 50: 1.02 + rank 51: 0.84 + rank 52: 0.88 + rank 53: 1.31 + rank 54: 0.94 + rank 55: 1.01 + rank 56: 0.87 + rank 57: 0.88 + rank 58: 1.09 + rank 59: 1.20 + rank 60: 0.81 + rank 61: 0.85 + rank 62: 1.18 + rank 63: 1.04 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.03 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.26 + rank 1: 1.92 + rank 2: 1.81 + rank 3: 2.23 + rank 4: 1.73 + rank 5: 2.08 + rank 6: 1.99 + rank 7: 2.06 + rank 8: 1.90 + rank 9: 2.20 + rank 10: 1.82 + rank 11: 1.77 + rank 12: 1.87 + rank 13: 1.76 + rank 14: 1.94 + rank 15: 2.10 + rank 16: 2.56 + rank 17: 2.28 + rank 18: 2.32 + rank 19: 2.32 + rank 20: 2.21 + rank 21: 2.39 + rank 22: 2.61 + rank 23: 2.39 + rank 24: 2.11 + rank 25: 2.05 + rank 26: 2.07 + rank 27: 2.27 + rank 28: 2.45 + rank 29: 2.26 + rank 30: 2.12 + rank 31: 2.27 + rank 32: 2.08 + rank 33: 2.04 + rank 34: 1.99 + rank 35: 2.44 + rank 36: 2.12 + rank 37: 2.10 + rank 38: 2.28 + rank 39: 1.98 + rank 40: 1.77 + rank 41: 1.93 + rank 42: 1.84 + rank 43: 2.03 + rank 44: 1.88 + rank 45: 2.06 + rank 46: 1.81 + rank 47: 1.67 + rank 48: 2.00 + rank 49: 1.80 + rank 50: 2.44 + rank 51: 1.74 + rank 52: 1.95 + rank 53: 1.83 + rank 54: 1.81 + rank 55: 1.83 + rank 56: 1.84 + rank 57: 2.45 + rank 58: 1.83 + rank 59: 1.76 + rank 60: 1.76 + rank 61: 1.68 + rank 62: 1.86 + rank 63: 1.71 + optimizer-inner-step: + rank 0: 42.61 + rank 1: 42.57 + rank 2: 42.28 + rank 3: 42.47 + rank 4: 42.23 + rank 5: 42.39 + rank 6: 42.73 + rank 7: 42.72 + rank 8: 42.55 + rank 9: 42.50 + rank 10: 42.32 + rank 11: 42.52 + rank 12: 42.48 + rank 13: 42.73 + rank 14: 42.49 + rank 15: 42.32 + rank 16: 42.97 + rank 17: 42.50 + rank 18: 42.67 + rank 19: 42.54 + rank 20: 42.49 + rank 21: 42.42 + rank 22: 43.10 + rank 23: 42.59 + rank 24: 42.07 + rank 25: 42.54 + rank 26: 42.51 + rank 27: 42.59 + rank 28: 46.28 + rank 29: 42.30 + rank 30: 42.44 + rank 31: 42.62 + rank 32: 42.55 + rank 33: 42.38 + rank 34: 42.52 + rank 35: 49.71 + rank 36: 42.70 + rank 37: 42.55 + rank 38: 42.30 + rank 39: 42.38 + rank 40: 42.41 + rank 41: 42.59 + rank 42: 42.62 + rank 43: 42.70 + rank 44: 42.29 + rank 45: 42.93 + rank 46: 42.59 + rank 47: 42.23 + rank 48: 42.43 + rank 49: 42.35 + rank 50: 54.40 + rank 51: 42.17 + rank 52: 42.48 + rank 53: 42.35 + rank 54: 42.56 + rank 55: 42.52 + rank 56: 42.20 + rank 57: 46.76 + rank 58: 42.44 + rank 59: 42.32 + rank 60: 42.62 + rank 61: 42.30 + rank 62: 42.44 + rank 63: 42.30 + optimizer-copy-main-to-model-params: + rank 0: 4.42 + rank 1: 3.85 + rank 2: 3.76 + rank 3: 4.34 + rank 4: 3.69 + rank 5: 4.02 + rank 6: 4.06 + rank 7: 4.03 + rank 8: 3.84 + rank 9: 4.48 + rank 10: 3.83 + rank 11: 3.80 + rank 12: 3.91 + rank 13: 3.83 + rank 14: 4.01 + rank 15: 4.24 + rank 16: 4.48 + rank 17: 4.08 + rank 18: 4.09 + rank 19: 4.12 + rank 20: 4.10 + rank 21: 4.28 + rank 22: 4.69 + rank 23: 4.25 + rank 24: 4.02 + rank 25: 4.15 + rank 26: 3.99 + rank 27: 4.02 + rank 28: 4.37 + rank 29: 4.39 + rank 30: 4.37 + rank 31: 4.03 + rank 32: 4.13 + rank 33: 3.94 + rank 34: 4.03 + rank 35: 4.55 + rank 36: 4.28 + rank 37: 4.15 + rank 38: 4.48 + rank 39: 3.90 + rank 40: 3.79 + rank 41: 3.98 + rank 42: 3.95 + rank 43: 4.03 + rank 44: 4.00 + rank 45: 4.25 + rank 46: 3.84 + rank 47: 3.72 + rank 48: 4.09 + rank 49: 3.88 + rank 50: 4.96 + rank 51: 3.97 + rank 52: 4.20 + rank 53: 3.86 + rank 54: 3.89 + rank 55: 3.85 + rank 56: 3.78 + rank 57: 4.76 + rank 58: 3.95 + rank 59: 3.76 + rank 60: 3.83 + rank 61: 3.75 + rank 62: 3.89 + rank 63: 3.65 + optimizer: + rank 0: 54.52 + rank 1: 53.64 + rank 2: 53.16 + rank 3: 54.22 + rank 4: 53.03 + rank 5: 53.66 + rank 6: 54.11 + rank 7: 54.04 + rank 8: 53.59 + rank 9: 54.39 + rank 10: 53.34 + rank 11: 53.48 + rank 12: 53.61 + rank 13: 53.73 + rank 14: 53.75 + rank 15: 53.87 + rank 16: 55.13 + rank 17: 54.06 + rank 18: 54.21 + rank 19: 54.13 + rank 20: 54.07 + rank 21: 54.28 + rank 22: 55.42 + rank 23: 54.50 + rank 24: 53.55 + rank 25: 54.01 + rank 26: 53.68 + rank 27: 53.82 + rank 28: 58.03 + rank 29: 54.09 + rank 30: 54.13 + rank 31: 53.89 + rank 32: 53.93 + rank 33: 53.46 + rank 34: 53.66 + rank 35: 61.65 + rank 36: 54.20 + rank 37: 53.90 + rank 38: 54.13 + rank 39: 53.42 + rank 40: 53.36 + rank 41: 53.79 + rank 42: 53.80 + rank 43: 53.92 + rank 44: 53.51 + rank 45: 54.47 + rank 46: 53.58 + rank 47: 53.08 + rank 48: 53.88 + rank 49: 53.45 + rank 50: 67.00 + rank 51: 53.39 + rank 52: 54.03 + rank 53: 53.42 + rank 54: 53.63 + rank 55: 53.55 + rank 56: 53.14 + rank 57: 59.12 + rank 58: 53.60 + rank 59: 53.18 + rank 60: 53.56 + rank 61: 53.16 + rank 62: 53.49 + rank 63: 53.08 + interval-time: + rank 0: 0.83 + rank 1: 0.83 + rank 2: 0.79 + rank 3: 0.76 + rank 4: 0.76 + rank 5: 0.90 + rank 6: 0.78 + rank 7: 0.83 + rank 8: 0.84 + rank 9: 0.85 + rank 10: 0.85 + rank 11: 0.84 + rank 12: 0.85 + rank 13: 0.73 + rank 14: 0.88 + rank 15: 0.83 + rank 16: 0.85 + rank 17: 0.84 + rank 18: 0.89 + rank 19: 0.88 + rank 20: 0.83 + rank 21: 0.83 + rank 22: 0.81 + rank 23: 0.74 + rank 24: 1.33 + rank 25: 1.37 + rank 26: 1.13 + rank 27: 1.14 + rank 28: 1.32 + rank 29: 1.39 + rank 30: 1.23 + rank 31: 1.39 + rank 32: 0.95 + rank 33: 1.00 + rank 34: 0.96 + rank 35: 0.88 + rank 36: 0.82 + rank 37: 0.92 + rank 38: 0.82 + rank 39: 0.82 + rank 40: 0.85 + rank 41: 0.83 + rank 42: 0.83 + rank 43: 0.75 + rank 44: 0.73 + rank 45: 0.80 + rank 46: 0.81 + rank 47: 0.82 + rank 48: 1.31 + rank 49: 1.22 + rank 50: 1.04 + rank 51: 1.18 + rank 52: 1.26 + rank 53: 1.23 + rank 54: 1.22 + rank 55: 1.29 + rank 56: 0.84 + rank 57: 0.74 + rank 58: 0.75 + rank 59: 0.75 + rank 60: 0.76 + rank 61: 0.74 + rank 62: 0.74 + rank 63: 1.31 + [2026-10-14 04:56:46] iteration 1003/ 2384 | consumed samples: 32096 | elapsed time per iteration (ms): 222193.3 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75507.1 | valid tokens/s: 75507.1 | physical tokens/s: 76410.6 | loss tokens/s: 54623.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.574635E-01 | lm-global-supervised-tokens: 1.213704E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.325664E-02 | lm-token-buffer-mib: 2.330368E+02 | global_load_balancing_loss: 1.000214E+00 | loss scale: 1.0 | grad norm: 0.224 | params norm: 11186.575 | params rms: 0.060572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:14:08.979579 | finish at 2026-10-17 18:10:55 +times across ranks (ms): + forward-backward: + rank 0: 221943.52 + rank 1: 221942.53 + rank 2: 221942.23 + rank 3: 221942.30 + rank 4: 221944.88 + rank 5: 221941.88 + rank 6: 221939.75 + rank 7: 221944.41 + rank 8: 221941.16 + rank 9: 221943.22 + rank 10: 221940.91 + rank 11: 221940.19 + rank 12: 221943.59 + rank 13: 221944.17 + rank 14: 221941.75 + rank 15: 221944.62 + rank 16: 221937.97 + rank 17: 221941.56 + rank 18: 221939.66 + rank 19: 221943.09 + rank 20: 221942.69 + rank 21: 221943.77 + rank 22: 221937.16 + rank 23: 221939.44 + rank 24: 221935.86 + rank 25: 221937.52 + rank 26: 221936.14 + rank 27: 221937.25 + rank 28: 221937.70 + rank 29: 221937.38 + rank 30: 221936.70 + rank 31: 221939.09 + rank 32: 221933.20 + rank 33: 221932.70 + rank 34: 221931.77 + rank 35: 221931.88 + rank 36: 221934.34 + rank 37: 221931.48 + rank 38: 221928.97 + rank 39: 221934.16 + rank 40: 221930.39 + rank 41: 221931.23 + rank 42: 221928.09 + rank 43: 221928.06 + rank 44: 221929.48 + rank 45: 221928.86 + rank 46: 221933.11 + rank 47: 221934.41 + rank 48: 221936.06 + rank 49: 221938.45 + rank 50: 221935.59 + rank 51: 221937.78 + rank 52: 221935.97 + rank 53: 221937.78 + rank 54: 221938.06 + rank 55: 221940.34 + rank 56: 221939.31 + rank 57: 221940.56 + rank 58: 221939.31 + rank 59: 221940.00 + rank 60: 221939.53 + rank 61: 221938.70 + rank 62: 221941.05 + rank 63: 221943.33 + forward-compute: + rank 0: 127795.84 + rank 1: 127789.41 + rank 2: 127788.84 + rank 3: 127789.59 + rank 4: 127794.00 + rank 5: 127789.69 + rank 6: 127789.99 + rank 7: 127797.53 + rank 8: 127888.63 + rank 9: 127893.36 + rank 10: 127892.52 + rank 11: 127888.49 + rank 12: 127887.97 + rank 13: 127894.02 + rank 14: 127889.82 + rank 15: 127894.43 + rank 16: 127802.92 + rank 17: 127803.64 + rank 18: 127800.06 + rank 19: 127802.96 + rank 20: 127804.16 + rank 21: 127804.61 + rank 22: 127804.92 + rank 23: 127808.19 + rank 24: 127808.99 + rank 25: 127800.45 + rank 26: 127799.71 + rank 27: 127808.77 + rank 28: 127815.87 + rank 29: 127811.02 + rank 30: 127812.54 + rank 31: 127699.66 + rank 32: 61394.85 + rank 33: 61365.52 + rank 34: 61396.08 + rank 35: 61430.98 + rank 36: 61432.14 + rank 37: 61446.71 + rank 38: 61429.28 + rank 39: 61440.59 + rank 40: 61461.43 + rank 41: 61421.65 + rank 42: 61422.36 + rank 43: 61441.13 + rank 44: 61428.31 + rank 45: 61437.22 + rank 46: 61398.79 + rank 47: 61443.41 + rank 48: 61432.41 + rank 49: 61412.94 + rank 50: 61414.62 + rank 51: 61421.82 + rank 52: 61437.77 + rank 53: 61435.82 + rank 54: 61397.61 + rank 55: 61430.79 + rank 56: 61407.18 + rank 57: 61430.04 + rank 58: 61419.11 + rank 59: 61399.36 + rank 60: 61417.81 + rank 61: 61427.32 + rank 62: 61407.09 + rank 63: 61460.62 + backward-compute: + rank 0: 94090.20 + rank 1: 94095.18 + rank 2: 94092.84 + rank 3: 94093.08 + rank 4: 94091.21 + rank 5: 94095.46 + rank 6: 94090.09 + rank 7: 94088.88 + rank 8: 93998.31 + rank 9: 93991.84 + rank 10: 93991.19 + rank 11: 93997.44 + rank 12: 94001.70 + rank 13: 93996.04 + rank 14: 93996.20 + rank 15: 93995.98 + rank 16: 94085.16 + rank 17: 94086.61 + rank 18: 94085.98 + rank 19: 94086.60 + rank 20: 94087.28 + rank 21: 94086.09 + rank 22: 94083.60 + rank 23: 94083.30 + rank 24: 94032.15 + rank 25: 94039.16 + rank 26: 94036.05 + rank 27: 94027.00 + rank 28: 94024.88 + rank 29: 94030.25 + rank 30: 94026.84 + rank 31: 94137.95 + rank 32: 160493.11 + rank 33: 160521.98 + rank 34: 160491.62 + rank 35: 160454.84 + rank 36: 160458.64 + rank 37: 160440.61 + rank 38: 160456.03 + rank 39: 160449.91 + rank 40: 160434.86 + rank 41: 160476.44 + rank 42: 160471.64 + rank 43: 160452.19 + rank 44: 160467.53 + rank 45: 160458.89 + rank 46: 160499.98 + rank 47: 160456.50 + rank 48: 160469.00 + rank 49: 160490.05 + rank 50: 160484.36 + rank 51: 160485.34 + rank 52: 160463.50 + rank 53: 160468.33 + rank 54: 160503.41 + rank 55: 160473.91 + rank 56: 160499.98 + rank 57: 160475.52 + rank 58: 160486.09 + rank 59: 160505.38 + rank 60: 160487.59 + rank 61: 160475.62 + rank 62: 160500.41 + rank 63: 160448.91 + batch-generator: + rank 0: 45.89 + rank 1: 51.04 + rank 2: 50.33 + rank 3: 49.37 + rank 4: 48.88 + rank 5: 49.28 + rank 6: 48.73 + rank 7: 46.46 + rank 8: 67.28 + rank 9: 68.03 + rank 10: 70.56 + rank 11: 68.54 + rank 12: 67.79 + rank 13: 67.47 + rank 14: 71.07 + rank 15: 66.80 + rank 16: 36.25 + rank 17: 36.47 + rank 18: 36.22 + rank 19: 28.96 + rank 20: 36.32 + rank 21: 37.34 + rank 22: 37.11 + rank 23: 33.83 + rank 24: 49.46 + rank 25: 55.68 + rank 26: 57.97 + rank 27: 59.99 + rank 28: 59.44 + rank 29: 58.42 + rank 30: 52.00 + rank 31: 52.67 + rank 32: 57.35 + rank 33: 63.47 + rank 34: 60.62 + rank 35: 56.06 + rank 36: 51.60 + rank 37: 52.01 + rank 38: 41.44 + rank 39: 54.18 + rank 40: 34.33 + rank 41: 35.16 + rank 42: 33.91 + rank 43: 36.22 + rank 44: 33.01 + rank 45: 33.49 + rank 46: 35.65 + rank 47: 30.35 + rank 48: 38.00 + rank 49: 45.36 + rank 50: 43.66 + rank 51: 40.69 + rank 52: 44.51 + rank 53: 40.96 + rank 54: 37.57 + rank 55: 36.35 + rank 56: 32.54 + rank 57: 38.04 + rank 58: 36.06 + rank 59: 35.49 + rank 60: 35.28 + rank 61: 40.20 + rank 62: 28.55 + rank 63: 31.87 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.04 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.03 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.04 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.80 + rank 1: 1.06 + rank 2: 0.87 + rank 3: 0.87 + rank 4: 1.24 + rank 5: 0.89 + rank 6: 0.80 + rank 7: 0.84 + rank 8: 0.84 + rank 9: 1.04 + rank 10: 0.88 + rank 11: 0.86 + rank 12: 1.21 + rank 13: 0.93 + rank 14: 1.27 + rank 15: 0.81 + rank 16: 1.27 + rank 17: 0.85 + rank 18: 1.32 + rank 19: 0.94 + rank 20: 0.95 + rank 21: 0.95 + rank 22: 0.90 + rank 23: 1.03 + rank 24: 1.54 + rank 25: 1.27 + rank 26: 1.32 + rank 27: 1.45 + rank 28: 1.72 + rank 29: 1.64 + rank 30: 1.26 + rank 31: 1.44 + rank 32: 1.47 + rank 33: 1.37 + rank 34: 1.60 + rank 35: 1.69 + rank 36: 1.45 + rank 37: 1.29 + rank 38: 1.49 + rank 39: 1.19 + rank 40: 1.02 + rank 41: 1.02 + rank 42: 1.08 + rank 43: 0.80 + rank 44: 0.79 + rank 45: 0.96 + rank 46: 1.19 + rank 47: 1.02 + rank 48: 1.09 + rank 49: 1.31 + rank 50: 1.29 + rank 51: 1.03 + rank 52: 1.11 + rank 53: 1.26 + rank 54: 1.21 + rank 55: 1.38 + rank 56: 0.94 + rank 57: 1.13 + rank 58: 1.10 + rank 59: 1.26 + rank 60: 1.03 + rank 61: 0.93 + rank 62: 0.93 + rank 63: 0.86 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.03 + rank 36: 0.02 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.02 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.25 + rank 1: 1.86 + rank 2: 1.79 + rank 3: 2.17 + rank 4: 1.68 + rank 5: 1.95 + rank 6: 1.94 + rank 7: 1.98 + rank 8: 1.91 + rank 9: 2.16 + rank 10: 1.83 + rank 11: 1.75 + rank 12: 1.87 + rank 13: 1.73 + rank 14: 1.93 + rank 15: 2.09 + rank 16: 2.55 + rank 17: 2.24 + rank 18: 2.26 + rank 19: 2.25 + rank 20: 2.32 + rank 21: 2.41 + rank 22: 2.72 + rank 23: 2.42 + rank 24: 1.97 + rank 25: 2.05 + rank 26: 2.08 + rank 27: 2.19 + rank 28: 2.32 + rank 29: 2.17 + rank 30: 2.18 + rank 31: 2.23 + rank 32: 2.29 + rank 33: 1.95 + rank 34: 2.10 + rank 35: 2.43 + rank 36: 2.15 + rank 37: 2.10 + rank 38: 2.27 + rank 39: 2.14 + rank 40: 1.78 + rank 41: 1.90 + rank 42: 1.83 + rank 43: 1.93 + rank 44: 1.89 + rank 45: 2.09 + rank 46: 1.75 + rank 47: 1.70 + rank 48: 1.92 + rank 49: 1.85 + rank 50: 2.52 + rank 51: 1.76 + rank 52: 1.97 + rank 53: 1.79 + rank 54: 1.81 + rank 55: 1.85 + rank 56: 1.81 + rank 57: 2.38 + rank 58: 1.77 + rank 59: 1.83 + rank 60: 1.76 + rank 61: 1.70 + rank 62: 1.83 + rank 63: 1.75 + optimizer-inner-step: + rank 0: 46.83 + rank 1: 42.69 + rank 2: 42.48 + rank 3: 42.61 + rank 4: 42.31 + rank 5: 42.46 + rank 6: 42.84 + rank 7: 42.76 + rank 8: 42.67 + rank 9: 42.61 + rank 10: 42.51 + rank 11: 42.62 + rank 12: 42.61 + rank 13: 42.91 + rank 14: 42.60 + rank 15: 42.43 + rank 16: 42.89 + rank 17: 42.69 + rank 18: 42.79 + rank 19: 42.63 + rank 20: 42.56 + rank 21: 42.50 + rank 22: 43.18 + rank 23: 42.76 + rank 24: 42.37 + rank 25: 42.67 + rank 26: 42.55 + rank 27: 42.80 + rank 28: 46.36 + rank 29: 42.73 + rank 30: 42.60 + rank 31: 42.77 + rank 32: 42.60 + rank 33: 42.51 + rank 34: 42.62 + rank 35: 49.07 + rank 36: 42.80 + rank 37: 42.64 + rank 38: 42.52 + rank 39: 42.45 + rank 40: 42.55 + rank 41: 42.76 + rank 42: 42.70 + rank 43: 42.88 + rank 44: 42.39 + rank 45: 43.08 + rank 46: 42.69 + rank 47: 42.30 + rank 48: 42.54 + rank 49: 42.50 + rank 50: 54.57 + rank 51: 42.34 + rank 52: 42.72 + rank 53: 42.45 + rank 54: 42.66 + rank 55: 42.61 + rank 56: 42.29 + rank 57: 46.80 + rank 58: 42.58 + rank 59: 42.43 + rank 60: 42.68 + rank 61: 42.42 + rank 62: 42.49 + rank 63: 42.37 + optimizer-copy-main-to-model-params: + rank 0: 4.41 + rank 1: 3.83 + rank 2: 3.75 + rank 3: 4.40 + rank 4: 3.73 + rank 5: 3.98 + rank 6: 4.00 + rank 7: 4.02 + rank 8: 3.91 + rank 9: 4.46 + rank 10: 3.86 + rank 11: 3.80 + rank 12: 3.99 + rank 13: 3.82 + rank 14: 4.04 + rank 15: 4.28 + rank 16: 4.38 + rank 17: 4.02 + rank 18: 4.03 + rank 19: 4.07 + rank 20: 4.11 + rank 21: 4.35 + rank 22: 4.70 + rank 23: 4.20 + rank 24: 3.93 + rank 25: 4.18 + rank 26: 4.01 + rank 27: 4.12 + rank 28: 4.43 + rank 29: 4.47 + rank 30: 4.34 + rank 31: 4.20 + rank 32: 4.22 + rank 33: 3.98 + rank 34: 4.11 + rank 35: 4.60 + rank 36: 4.30 + rank 37: 4.16 + rank 38: 4.50 + rank 39: 3.96 + rank 40: 3.79 + rank 41: 3.91 + rank 42: 3.93 + rank 43: 4.02 + rank 44: 3.96 + rank 45: 4.23 + rank 46: 3.74 + rank 47: 3.72 + rank 48: 4.05 + rank 49: 3.89 + rank 50: 5.12 + rank 51: 3.95 + rank 52: 4.18 + rank 53: 3.88 + rank 54: 3.90 + rank 55: 3.88 + rank 56: 3.75 + rank 57: 4.75 + rank 58: 3.86 + rank 59: 3.75 + rank 60: 3.72 + rank 61: 3.70 + rank 62: 3.93 + rank 63: 3.67 + optimizer: + rank 0: 59.42 + rank 1: 54.39 + rank 2: 54.00 + rank 3: 55.04 + rank 4: 53.85 + rank 5: 54.33 + rank 6: 54.78 + rank 7: 54.78 + rank 8: 54.47 + rank 9: 55.18 + rank 10: 54.22 + rank 11: 54.22 + rank 12: 54.49 + rank 13: 54.56 + rank 14: 54.54 + rank 15: 54.71 + rank 16: 55.62 + rank 17: 54.83 + rank 18: 54.92 + rank 19: 54.87 + rank 20: 54.81 + rank 21: 55.07 + rank 22: 56.22 + rank 23: 55.23 + rank 24: 54.15 + rank 25: 54.78 + rank 26: 54.39 + rank 27: 54.75 + rank 28: 58.79 + rank 29: 55.23 + rank 30: 54.90 + rank 31: 54.85 + rank 32: 54.71 + rank 33: 54.29 + rank 34: 54.53 + rank 35: 61.74 + rank 36: 54.99 + rank 37: 54.68 + rank 38: 55.02 + rank 39: 54.22 + rank 40: 54.18 + rank 41: 54.53 + rank 42: 54.51 + rank 43: 54.80 + rank 44: 54.25 + rank 45: 55.28 + rank 46: 54.22 + rank 47: 53.85 + rank 48: 54.51 + rank 49: 54.19 + rank 50: 67.90 + rank 51: 54.09 + rank 52: 54.82 + rank 53: 54.17 + rank 54: 54.37 + rank 55: 54.31 + rank 56: 53.89 + rank 57: 59.76 + rank 58: 54.29 + rank 59: 53.96 + rank 60: 54.20 + rank 61: 53.92 + rank 62: 54.27 + rank 63: 53.85 + interval-time: + rank 0: 3.13 + rank 1: 3.15 + rank 2: 3.09 + rank 3: 3.07 + rank 4: 3.08 + rank 5: 3.18 + rank 6: 3.07 + rank 7: 3.13 + rank 8: 0.86 + rank 9: 0.90 + rank 10: 0.89 + rank 11: 0.83 + rank 12: 0.87 + rank 13: 0.74 + rank 14: 0.88 + rank 15: 0.84 + rank 16: 0.84 + rank 17: 0.83 + rank 18: 0.85 + rank 19: 0.87 + rank 20: 0.85 + rank 21: 0.84 + rank 22: 0.75 + rank 23: 0.75 + rank 24: 0.99 + rank 25: 0.97 + rank 26: 0.86 + rank 27: 0.96 + rank 28: 0.94 + rank 29: 0.98 + rank 30: 0.87 + rank 31: 0.99 + rank 32: 0.95 + rank 33: 0.99 + rank 34: 0.95 + rank 35: 0.85 + rank 36: 0.81 + rank 37: 0.86 + rank 38: 0.80 + rank 39: 0.81 + rank 40: 0.86 + rank 41: 0.83 + rank 42: 0.83 + rank 43: 0.76 + rank 44: 0.75 + rank 45: 0.82 + rank 46: 0.81 + rank 47: 0.82 + rank 48: 0.91 + rank 49: 0.86 + rank 50: 0.85 + rank 51: 0.82 + rank 52: 0.90 + rank 53: 0.80 + rank 54: 0.81 + rank 55: 0.91 + rank 56: 0.83 + rank 57: 0.74 + rank 58: 0.74 + rank 59: 0.74 + rank 60: 0.78 + rank 61: 0.74 + rank 62: 0.73 + rank 63: 1.49 + [2026-10-14 05:00:26] iteration 1004/ 2384 | consumed samples: 32128 | elapsed time per iteration (ms): 220817.3 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.72% | FLOPs estimator: workload-aware | logical tokens/s: 75977.8 | valid tokens/s: 75977.8 | physical tokens/s: 76343.0 | loss tokens/s: 60447.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.591077E-01 | lm-global-supervised-tokens: 1.334785E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.851918E-02 | lm-token-buffer-mib: 2.337662E+02 | global_load_balancing_loss: 1.000571E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11186.546 | params rms: 0.060572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:38:47.886286 | finish at 2026-10-17 17:39:14 +times across ranks (ms): + forward-backward: + rank 0: 220567.80 + rank 1: 220568.41 + rank 2: 220566.23 + rank 3: 220566.34 + rank 4: 220568.53 + rank 5: 220568.05 + rank 6: 220575.36 + rank 7: 220578.28 + rank 8: 220577.94 + rank 9: 220549.28 + rank 10: 220572.66 + rank 11: 220580.08 + rank 12: 220582.06 + rank 13: 220581.05 + rank 14: 220576.75 + rank 15: 220578.48 + rank 16: 220567.42 + rank 17: 220569.34 + rank 18: 220568.94 + rank 19: 220570.19 + rank 20: 220572.11 + rank 21: 220566.20 + rank 22: 220567.55 + rank 23: 220561.33 + rank 24: 220552.27 + rank 25: 220557.25 + rank 26: 220558.61 + rank 27: 220558.81 + rank 28: 220558.91 + rank 29: 220559.75 + rank 30: 220553.72 + rank 31: 220554.41 + rank 32: 220554.42 + rank 33: 220554.98 + rank 34: 220552.23 + rank 35: 220551.02 + rank 36: 220552.80 + rank 37: 220552.70 + rank 38: 220553.42 + rank 39: 220554.25 + rank 40: 220568.81 + rank 41: 220538.56 + rank 42: 220561.05 + rank 43: 220567.64 + rank 44: 220567.28 + rank 45: 220566.56 + rank 46: 220569.69 + rank 47: 220570.16 + rank 48: 220566.81 + rank 49: 220567.86 + rank 50: 220567.20 + rank 51: 220566.56 + rank 52: 220566.55 + rank 53: 220562.20 + rank 54: 220571.53 + rank 55: 220563.17 + rank 56: 220563.89 + rank 57: 220567.84 + rank 58: 220565.95 + rank 59: 220566.78 + rank 60: 220563.34 + rank 61: 220567.09 + rank 62: 220565.81 + rank 63: 220566.70 + forward-compute: + rank 0: 118777.51 + rank 1: 118778.57 + rank 2: 118768.91 + rank 3: 118763.95 + rank 4: 118766.95 + rank 5: 118778.59 + rank 6: 118776.59 + rank 7: 118787.39 + rank 8: 118784.07 + rank 9: 118761.90 + rank 10: 118785.73 + rank 11: 118788.73 + rank 12: 118796.63 + rank 13: 118785.11 + rank 14: 118779.23 + rank 15: 118798.23 + rank 16: 118764.60 + rank 17: 118763.76 + rank 18: 118763.65 + rank 19: 118758.92 + rank 20: 118758.26 + rank 21: 118753.28 + rank 22: 118762.52 + rank 23: 118752.12 + rank 24: 118741.57 + rank 25: 118747.56 + rank 26: 118559.56 + rank 27: 118755.78 + rank 28: 118758.27 + rank 29: 118755.62 + rank 30: 118755.16 + rank 31: 118753.84 + rank 32: 61748.42 + rank 33: 61779.32 + rank 34: 61763.50 + rank 35: 61764.67 + rank 36: 61814.88 + rank 37: 61809.86 + rank 38: 61801.33 + rank 39: 61799.09 + rank 40: 61830.95 + rank 41: 61773.32 + rank 42: 61785.57 + rank 43: 61797.98 + rank 44: 61790.07 + rank 45: 61811.42 + rank 46: 61794.98 + rank 47: 61817.28 + rank 48: 61771.49 + rank 49: 61784.42 + rank 50: 61760.16 + rank 51: 61765.26 + rank 52: 61818.30 + rank 53: 61799.00 + rank 54: 61773.14 + rank 55: 61776.18 + rank 56: 61776.25 + rank 57: 61777.64 + rank 58: 61775.64 + rank 59: 61766.84 + rank 60: 61787.36 + rank 61: 61792.20 + rank 62: 61757.93 + rank 63: 61800.58 + backward-compute: + rank 0: 101715.71 + rank 1: 101715.58 + rank 2: 101723.92 + rank 3: 101726.45 + rank 4: 101725.62 + rank 5: 101715.59 + rank 6: 101718.52 + rank 7: 101708.23 + rank 8: 101745.87 + rank 9: 101736.14 + rank 10: 101733.26 + rank 11: 101742.26 + rank 12: 101734.08 + rank 13: 101746.09 + rank 14: 101745.78 + rank 15: 101728.83 + rank 16: 101754.13 + rank 17: 101755.41 + rank 18: 101755.40 + rank 19: 101760.73 + rank 20: 101766.39 + rank 21: 101763.52 + rank 22: 101760.23 + rank 23: 101762.80 + rank 24: 101755.04 + rank 25: 101748.86 + rank 26: 101935.46 + rank 27: 101740.84 + rank 28: 101733.04 + rank 29: 101743.24 + rank 30: 101739.91 + rank 31: 101741.12 + rank 32: 158769.42 + rank 33: 158735.42 + rank 34: 158750.53 + rank 35: 158749.20 + rank 36: 158700.52 + rank 37: 158705.00 + rank 38: 158713.41 + rank 39: 158717.31 + rank 40: 158703.03 + rank 41: 158731.66 + rank 42: 158742.25 + rank 43: 158735.64 + rank 44: 158743.62 + rank 45: 158722.55 + rank 46: 158739.50 + rank 47: 158718.06 + rank 48: 158761.56 + rank 49: 158749.17 + rank 50: 158770.03 + rank 51: 158771.00 + rank 52: 158716.75 + rank 53: 158727.80 + rank 54: 158761.56 + rank 55: 158753.84 + rank 56: 158754.44 + rank 57: 158754.20 + rank 58: 158755.83 + rank 59: 158765.98 + rank 60: 158742.27 + rank 61: 158739.36 + rank 62: 158775.80 + rank 63: 158732.58 + batch-generator: + rank 0: 34.45 + rank 1: 36.58 + rank 2: 34.09 + rank 3: 33.64 + rank 4: 36.09 + rank 5: 36.37 + rank 6: 35.96 + rank 7: 35.06 + rank 8: 45.80 + rank 9: 44.14 + rank 10: 43.80 + rank 11: 44.32 + rank 12: 46.54 + rank 13: 43.05 + rank 14: 43.93 + rank 15: 44.98 + rank 16: 31.15 + rank 17: 34.82 + rank 18: 33.02 + rank 19: 27.47 + rank 20: 34.36 + rank 21: 34.39 + rank 22: 32.61 + rank 23: 32.07 + rank 24: 39.76 + rank 25: 44.63 + rank 26: 49.28 + rank 27: 45.54 + rank 28: 46.19 + rank 29: 44.47 + rank 30: 45.12 + rank 31: 46.62 + rank 32: 41.48 + rank 33: 50.77 + rank 34: 44.36 + rank 35: 43.02 + rank 36: 36.51 + rank 37: 40.25 + rank 38: 42.80 + rank 39: 40.34 + rank 40: 32.82 + rank 41: 33.43 + rank 42: 34.81 + rank 43: 35.84 + rank 44: 34.28 + rank 45: 35.65 + rank 46: 34.44 + rank 47: 34.39 + rank 48: 38.70 + rank 49: 39.07 + rank 50: 47.47 + rank 51: 38.47 + rank 52: 37.16 + rank 53: 39.46 + rank 54: 36.52 + rank 55: 31.67 + rank 56: 33.61 + rank 57: 36.31 + rank 58: 36.81 + rank 59: 36.24 + rank 60: 37.42 + rank 61: 35.89 + rank 62: 26.51 + rank 63: 30.12 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.02 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.04 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.04 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.04 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.04 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.80 + rank 1: 0.91 + rank 2: 1.07 + rank 3: 0.82 + rank 4: 1.09 + rank 5: 0.88 + rank 6: 1.43 + rank 7: 1.39 + rank 8: 0.90 + rank 9: 0.86 + rank 10: 0.89 + rank 11: 0.96 + rank 12: 1.34 + rank 13: 1.07 + rank 14: 1.11 + rank 15: 1.24 + rank 16: 0.92 + rank 17: 1.21 + rank 18: 1.01 + rank 19: 0.86 + rank 20: 1.12 + rank 21: 1.10 + rank 22: 0.85 + rank 23: 0.94 + rank 24: 1.29 + rank 25: 1.29 + rank 26: 1.30 + rank 27: 1.24 + rank 28: 1.49 + rank 29: 1.51 + rank 30: 1.55 + rank 31: 1.62 + rank 32: 1.41 + rank 33: 1.33 + rank 34: 1.93 + rank 35: 1.39 + rank 36: 1.18 + rank 37: 1.32 + rank 38: 1.21 + rank 39: 1.44 + rank 40: 1.12 + rank 41: 0.84 + rank 42: 0.80 + rank 43: 0.77 + rank 44: 0.84 + rank 45: 0.79 + rank 46: 0.99 + rank 47: 0.92 + rank 48: 1.09 + rank 49: 1.01 + rank 50: 1.51 + rank 51: 0.89 + rank 52: 0.99 + rank 53: 1.32 + rank 54: 1.56 + rank 55: 0.96 + rank 56: 1.20 + rank 57: 0.94 + rank 58: 1.12 + rank 59: 0.82 + rank 60: 1.00 + rank 61: 1.35 + rank 62: 1.01 + rank 63: 1.14 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.24 + rank 1: 1.86 + rank 2: 1.80 + rank 3: 2.16 + rank 4: 1.67 + rank 5: 1.87 + rank 6: 3.08 + rank 7: 3.14 + rank 8: 1.90 + rank 9: 2.21 + rank 10: 1.82 + rank 11: 1.75 + rank 12: 1.87 + rank 13: 1.69 + rank 14: 1.93 + rank 15: 2.15 + rank 16: 2.51 + rank 17: 2.24 + rank 18: 2.28 + rank 19: 2.31 + rank 20: 2.19 + rank 21: 2.31 + rank 22: 2.59 + rank 23: 2.39 + rank 24: 1.93 + rank 25: 2.16 + rank 26: 2.04 + rank 27: 2.13 + rank 28: 2.40 + rank 29: 2.30 + rank 30: 2.13 + rank 31: 2.22 + rank 32: 2.10 + rank 33: 2.05 + rank 34: 2.11 + rank 35: 2.39 + rank 36: 2.22 + rank 37: 2.24 + rank 38: 2.41 + rank 39: 2.05 + rank 40: 1.76 + rank 41: 1.93 + rank 42: 1.91 + rank 43: 1.91 + rank 44: 1.92 + rank 45: 2.09 + rank 46: 1.78 + rank 47: 1.76 + rank 48: 1.92 + rank 49: 1.79 + rank 50: 2.50 + rank 51: 1.73 + rank 52: 1.92 + rank 53: 1.77 + rank 54: 1.76 + rank 55: 1.80 + rank 56: 1.79 + rank 57: 2.49 + rank 58: 1.78 + rank 59: 1.75 + rank 60: 1.79 + rank 61: 1.70 + rank 62: 1.84 + rank 63: 1.73 + optimizer-inner-step: + rank 0: 43.07 + rank 1: 40.10 + rank 2: 39.83 + rank 3: 40.07 + rank 4: 39.78 + rank 5: 40.04 + rank 6: 43.56 + rank 7: 40.51 + rank 8: 42.44 + rank 9: 42.39 + rank 10: 42.23 + rank 11: 42.33 + rank 12: 42.38 + rank 13: 42.57 + rank 14: 42.40 + rank 15: 42.23 + rank 16: 42.77 + rank 17: 42.45 + rank 18: 42.53 + rank 19: 42.34 + rank 20: 42.35 + rank 21: 42.25 + rank 22: 42.89 + rank 23: 42.51 + rank 24: 42.06 + rank 25: 42.22 + rank 26: 42.24 + rank 27: 45.38 + rank 28: 46.82 + rank 29: 42.06 + rank 30: 42.10 + rank 31: 43.57 + rank 32: 42.37 + rank 33: 42.26 + rank 34: 42.35 + rank 35: 48.94 + rank 36: 42.51 + rank 37: 42.35 + rank 38: 42.20 + rank 39: 42.19 + rank 40: 42.36 + rank 41: 42.49 + rank 42: 42.51 + rank 43: 42.61 + rank 44: 42.16 + rank 45: 42.80 + rank 46: 42.44 + rank 47: 42.07 + rank 48: 42.39 + rank 49: 42.27 + rank 50: 54.37 + rank 51: 42.12 + rank 52: 42.44 + rank 53: 42.26 + rank 54: 42.49 + rank 55: 42.38 + rank 56: 42.03 + rank 57: 46.97 + rank 58: 42.30 + rank 59: 42.15 + rank 60: 42.52 + rank 61: 42.14 + rank 62: 42.27 + rank 63: 42.18 + optimizer-copy-main-to-model-params: + rank 0: 4.42 + rank 1: 3.87 + rank 2: 3.73 + rank 3: 4.41 + rank 4: 3.67 + rank 5: 3.96 + rank 6: 3.95 + rank 7: 4.01 + rank 8: 3.95 + rank 9: 4.58 + rank 10: 3.88 + rank 11: 3.81 + rank 12: 3.96 + rank 13: 3.78 + rank 14: 4.04 + rank 15: 4.24 + rank 16: 4.43 + rank 17: 4.01 + rank 18: 4.08 + rank 19: 4.05 + rank 20: 3.99 + rank 21: 4.25 + rank 22: 4.61 + rank 23: 4.19 + rank 24: 3.92 + rank 25: 4.35 + rank 26: 4.13 + rank 27: 4.26 + rank 28: 4.66 + rank 29: 4.51 + rank 30: 4.47 + rank 31: 4.10 + rank 32: 4.09 + rank 33: 3.93 + rank 34: 4.06 + rank 35: 4.56 + rank 36: 4.20 + rank 37: 4.05 + rank 38: 4.46 + rank 39: 3.89 + rank 40: 3.87 + rank 41: 4.05 + rank 42: 4.08 + rank 43: 4.11 + rank 44: 4.00 + rank 45: 4.27 + rank 46: 3.77 + rank 47: 3.80 + rank 48: 4.07 + rank 49: 3.88 + rank 50: 4.97 + rank 51: 3.97 + rank 52: 4.17 + rank 53: 3.87 + rank 54: 3.92 + rank 55: 3.88 + rank 56: 3.78 + rank 57: 4.75 + rank 58: 3.97 + rank 59: 3.79 + rank 60: 3.77 + rank 61: 3.69 + rank 62: 3.92 + rank 63: 3.68 + optimizer: + rank 0: 57.73 + rank 1: 53.94 + rank 2: 53.47 + rank 3: 54.66 + rank 4: 53.39 + rank 5: 53.98 + rank 6: 58.02 + rank 7: 55.10 + rank 8: 54.03 + rank 9: 54.88 + rank 10: 53.77 + rank 11: 53.75 + rank 12: 54.02 + rank 13: 53.96 + rank 14: 54.12 + rank 15: 54.24 + rank 16: 55.29 + rank 17: 54.38 + rank 18: 54.50 + rank 19: 54.35 + rank 20: 54.27 + rank 21: 54.52 + rank 22: 55.63 + rank 23: 54.78 + rank 24: 53.65 + rank 25: 54.47 + rank 26: 54.11 + rank 27: 57.36 + rank 28: 59.48 + rank 29: 54.55 + rank 30: 54.55 + rank 31: 55.37 + rank 32: 54.14 + rank 33: 53.78 + rank 34: 54.00 + rank 35: 61.34 + rank 36: 54.41 + rank 37: 54.06 + rank 38: 54.45 + rank 39: 53.67 + rank 40: 53.84 + rank 41: 54.19 + rank 42: 54.26 + rank 43: 54.39 + rank 44: 53.83 + rank 45: 54.85 + rank 46: 53.78 + rank 47: 53.45 + rank 48: 54.13 + rank 49: 53.74 + rank 50: 67.35 + rank 51: 53.69 + rank 52: 54.32 + rank 53: 53.74 + rank 54: 53.98 + rank 55: 53.84 + rank 56: 53.45 + rank 57: 59.73 + rank 58: 53.93 + rank 59: 53.53 + rank 60: 53.87 + rank 61: 53.41 + rank 62: 53.82 + rank 63: 53.47 + interval-time: + rank 0: 0.97 + rank 1: 1.00 + rank 2: 0.94 + rank 3: 0.92 + rank 4: 1.07 + rank 5: 1.05 + rank 6: 0.96 + rank 7: 1.01 + rank 8: 0.85 + rank 9: 0.85 + rank 10: 0.86 + rank 11: 0.84 + rank 12: 0.85 + rank 13: 0.74 + rank 14: 0.87 + rank 15: 0.86 + rank 16: 0.85 + rank 17: 0.84 + rank 18: 0.87 + rank 19: 0.88 + rank 20: 0.82 + rank 21: 0.82 + rank 22: 0.75 + rank 23: 0.73 + rank 24: 0.97 + rank 25: 0.95 + rank 26: 0.85 + rank 27: 0.96 + rank 28: 1.03 + rank 29: 0.98 + rank 30: 0.86 + rank 31: 0.98 + rank 32: 1.18 + rank 33: 1.32 + rank 34: 1.27 + rank 35: 1.09 + rank 36: 1.11 + rank 37: 1.19 + rank 38: 1.14 + rank 39: 1.18 + rank 40: 0.87 + rank 41: 0.84 + rank 42: 0.85 + rank 43: 0.76 + rank 44: 0.73 + rank 45: 0.82 + rank 46: 0.83 + rank 47: 0.84 + rank 48: 0.91 + rank 49: 0.86 + rank 50: 0.86 + rank 51: 0.81 + rank 52: 0.90 + rank 53: 0.81 + rank 54: 0.82 + rank 55: 0.90 + rank 56: 0.86 + rank 57: 0.76 + rank 58: 0.77 + rank 59: 0.79 + rank 60: 0.78 + rank 61: 0.76 + rank 62: 0.77 + rank 63: 1.34 + [2026-10-14 05:04:13] iteration 1005/ 2384 | consumed samples: 32160 | elapsed time per iteration (ms): 226128.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 74193.2 | valid tokens/s: 74193.2 | physical tokens/s: 75193.9 | loss tokens/s: 55426.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.115333E-01 | lm-global-supervised-tokens: 1.253340E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.337362E-02 | lm-token-buffer-mib: 2.335799E+02 | global_load_balancing_loss: 9.999340E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11186.516 | params rms: 0.060571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:37:10.721461 | finish at 2026-10-17 19:41:23 +times across ranks (ms): + forward-backward: + rank 0: 225883.95 + rank 1: 225886.86 + rank 2: 225885.53 + rank 3: 225882.33 + rank 4: 225888.53 + rank 5: 225886.25 + rank 6: 225879.95 + rank 7: 225883.66 + rank 8: 225893.84 + rank 9: 225894.86 + rank 10: 225895.34 + rank 11: 225896.70 + rank 12: 225897.19 + rank 13: 225899.08 + rank 14: 225893.64 + rank 15: 225895.17 + rank 16: 225883.36 + rank 17: 225887.86 + rank 18: 225888.12 + rank 19: 225875.28 + rank 20: 225899.23 + rank 21: 225900.53 + rank 22: 225884.59 + rank 23: 225885.08 + rank 24: 225877.72 + rank 25: 225882.38 + rank 26: 225884.06 + rank 27: 225883.27 + rank 28: 225882.62 + rank 29: 225881.58 + rank 30: 225878.55 + rank 31: 225880.17 + rank 32: 225870.66 + rank 33: 225874.98 + rank 34: 225871.61 + rank 35: 225866.97 + rank 36: 225875.42 + rank 37: 225870.81 + rank 38: 225866.78 + rank 39: 225872.75 + rank 40: 225887.91 + rank 41: 225887.23 + rank 42: 225886.66 + rank 43: 225887.03 + rank 44: 225888.05 + rank 45: 225887.02 + rank 46: 225887.94 + rank 47: 225888.61 + rank 48: 225882.98 + rank 49: 225887.45 + rank 50: 225884.05 + rank 51: 225873.27 + rank 52: 225883.33 + rank 53: 225885.08 + rank 54: 225885.31 + rank 55: 225885.95 + rank 56: 225886.55 + rank 57: 225888.00 + rank 58: 225889.70 + rank 59: 225888.25 + rank 60: 225885.78 + rank 61: 225885.44 + rank 62: 225887.14 + rank 63: 225889.31 + forward-compute: + rank 0: 130875.39 + rank 1: 130872.51 + rank 2: 130868.30 + rank 3: 130865.03 + rank 4: 130870.98 + rank 5: 130867.87 + rank 6: 130869.79 + rank 7: 130874.18 + rank 8: 130949.15 + rank 9: 130951.85 + rank 10: 130957.10 + rank 11: 130950.97 + rank 12: 130955.42 + rank 13: 130965.43 + rank 14: 130941.19 + rank 15: 130953.20 + rank 16: 130874.67 + rank 17: 130874.75 + rank 18: 130871.12 + rank 19: 130864.52 + rank 20: 130875.94 + rank 21: 130879.65 + rank 22: 130882.80 + rank 23: 130882.50 + rank 24: 130882.10 + rank 25: 130882.13 + rank 26: 130892.41 + rank 27: 130890.42 + rank 28: 130887.08 + rank 29: 130883.37 + rank 30: 130889.06 + rank 31: 130886.07 + rank 32: 61572.04 + rank 33: 61591.63 + rank 34: 61599.48 + rank 35: 61615.38 + rank 36: 61635.86 + rank 37: 61634.29 + rank 38: 61612.77 + rank 39: 61619.16 + rank 40: 61670.98 + rank 41: 61643.32 + rank 42: 61648.50 + rank 43: 61660.82 + rank 44: 61645.52 + rank 45: 61658.84 + rank 46: 61620.07 + rank 47: 61659.91 + rank 48: 61627.00 + rank 49: 61622.99 + rank 50: 61628.82 + rank 51: 61616.28 + rank 52: 61653.68 + rank 53: 61639.59 + rank 54: 61606.59 + rank 55: 61613.41 + rank 56: 61629.70 + rank 57: 61608.16 + rank 58: 61600.32 + rank 59: 61616.07 + rank 60: 61610.99 + rank 61: 61643.02 + rank 62: 61586.53 + rank 63: 61644.88 + backward-compute: + rank 0: 94957.87 + rank 1: 94963.87 + rank 2: 94962.67 + rank 3: 94963.44 + rank 4: 94961.21 + rank 5: 94965.22 + rank 6: 94959.77 + rank 7: 94961.09 + rank 8: 94896.29 + rank 9: 94889.04 + rank 10: 94885.66 + rank 11: 94894.52 + rank 12: 94887.79 + rank 13: 94880.45 + rank 14: 94897.06 + rank 15: 94889.59 + rank 16: 94959.80 + rank 17: 94961.38 + rank 18: 94963.16 + rank 19: 94958.77 + rank 20: 94960.98 + rank 21: 94957.68 + rank 22: 94955.49 + rank 23: 94952.95 + rank 24: 94904.02 + rank 25: 94900.63 + rank 26: 94888.80 + rank 27: 94891.73 + rank 28: 94896.04 + rank 29: 94902.38 + rank 30: 94894.53 + rank 31: 94893.57 + rank 32: 164260.30 + rank 33: 164242.69 + rank 34: 164234.39 + rank 35: 164213.73 + rank 36: 164201.38 + rank 37: 164198.91 + rank 38: 164213.38 + rank 39: 164215.75 + rank 40: 164178.94 + rank 41: 164209.27 + rank 42: 164205.20 + rank 43: 164193.00 + rank 44: 164207.41 + rank 45: 164195.05 + rank 46: 164232.64 + rank 47: 164193.83 + rank 48: 164219.94 + rank 49: 164228.98 + rank 50: 164218.66 + rank 51: 164223.23 + rank 52: 164197.80 + rank 53: 164211.16 + rank 54: 164241.92 + rank 55: 164240.56 + rank 56: 164224.67 + rank 57: 164245.11 + rank 58: 164254.53 + rank 59: 164238.17 + rank 60: 164240.00 + rank 61: 164208.56 + rank 62: 164267.89 + rank 63: 164211.09 + batch-generator: + rank 0: 32.72 + rank 1: 35.79 + rank 2: 36.22 + rank 3: 34.93 + rank 4: 36.61 + rank 5: 37.15 + rank 6: 37.49 + rank 7: 35.66 + rank 8: 57.70 + rank 9: 60.26 + rank 10: 63.49 + rank 11: 61.22 + rank 12: 60.36 + rank 13: 61.81 + rank 14: 57.20 + rank 15: 60.19 + rank 16: 34.63 + rank 17: 37.45 + rank 18: 35.39 + rank 19: 28.03 + rank 20: 35.88 + rank 21: 35.45 + rank 22: 33.76 + rank 23: 34.27 + rank 24: 41.07 + rank 25: 46.71 + rank 26: 52.83 + rank 27: 54.62 + rank 28: 49.28 + rank 29: 49.28 + rank 30: 47.67 + rank 31: 47.45 + rank 32: 44.42 + rank 33: 50.35 + rank 34: 41.37 + rank 35: 40.49 + rank 36: 37.81 + rank 37: 39.84 + rank 38: 41.15 + rank 39: 40.27 + rank 40: 33.85 + rank 41: 35.03 + rank 42: 34.59 + rank 43: 35.20 + rank 44: 32.75 + rank 45: 32.00 + rank 46: 34.03 + rank 47: 31.14 + rank 48: 38.86 + rank 49: 37.65 + rank 50: 45.51 + rank 51: 43.87 + rank 52: 39.64 + rank 53: 39.58 + rank 54: 35.13 + rank 55: 30.90 + rank 56: 31.89 + rank 57: 35.77 + rank 58: 38.75 + rank 59: 35.79 + rank 60: 35.81 + rank 61: 35.82 + rank 62: 27.11 + rank 63: 29.02 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.02 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.04 + rank 19: 0.03 + rank 20: 0.04 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.04 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.03 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.11 + rank 1: 1.04 + rank 2: 0.81 + rank 3: 0.82 + rank 4: 0.88 + rank 5: 1.19 + rank 6: 0.79 + rank 7: 1.07 + rank 8: 0.87 + rank 9: 1.02 + rank 10: 0.96 + rank 11: 1.26 + rank 12: 0.87 + rank 13: 1.01 + rank 14: 0.81 + rank 15: 1.04 + rank 16: 0.93 + rank 17: 0.92 + rank 18: 1.22 + rank 19: 0.94 + rank 20: 1.53 + rank 21: 1.61 + rank 22: 1.29 + rank 23: 0.91 + rank 24: 1.54 + rank 25: 1.33 + rank 26: 1.37 + rank 27: 1.28 + rank 28: 1.31 + rank 29: 1.46 + rank 30: 1.21 + rank 31: 1.67 + rank 32: 1.22 + rank 33: 1.81 + rank 34: 1.57 + rank 35: 1.29 + rank 36: 1.37 + rank 37: 1.27 + rank 38: 1.02 + rank 39: 1.29 + rank 40: 1.24 + rank 41: 1.05 + rank 42: 0.84 + rank 43: 0.88 + rank 44: 0.81 + rank 45: 0.88 + rank 46: 1.13 + rank 47: 1.09 + rank 48: 1.33 + rank 49: 1.37 + rank 50: 1.38 + rank 51: 0.98 + rank 52: 0.95 + rank 53: 0.92 + rank 54: 1.02 + rank 55: 0.86 + rank 56: 1.09 + rank 57: 0.93 + rank 58: 0.83 + rank 59: 1.14 + rank 60: 0.87 + rank 61: 0.89 + rank 62: 1.04 + rank 63: 0.84 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.03 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.03 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.03 + rank 37: 0.02 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.02 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.21 + rank 1: 1.83 + rank 2: 1.86 + rank 3: 2.35 + rank 4: 1.71 + rank 5: 1.97 + rank 6: 1.94 + rank 7: 2.07 + rank 8: 1.85 + rank 9: 2.23 + rank 10: 1.81 + rank 11: 1.75 + rank 12: 1.90 + rank 13: 1.75 + rank 14: 1.91 + rank 15: 2.26 + rank 16: 2.61 + rank 17: 2.25 + rank 18: 2.32 + rank 19: 2.26 + rank 20: 3.65 + rank 21: 3.75 + rank 22: 2.70 + rank 23: 2.44 + rank 24: 2.00 + rank 25: 2.14 + rank 26: 2.20 + rank 27: 2.26 + rank 28: 2.32 + rank 29: 2.16 + rank 30: 2.36 + rank 31: 2.20 + rank 32: 2.08 + rank 33: 1.93 + rank 34: 1.92 + rank 35: 2.23 + rank 36: 2.17 + rank 37: 2.08 + rank 38: 2.41 + rank 39: 2.08 + rank 40: 1.82 + rank 41: 1.98 + rank 42: 1.84 + rank 43: 1.91 + rank 44: 1.99 + rank 45: 2.12 + rank 46: 1.74 + rank 47: 1.72 + rank 48: 1.91 + rank 49: 1.85 + rank 50: 2.52 + rank 51: 1.84 + rank 52: 1.96 + rank 53: 1.75 + rank 54: 1.73 + rank 55: 1.81 + rank 56: 1.83 + rank 57: 2.44 + rank 58: 1.82 + rank 59: 1.80 + rank 60: 1.79 + rank 61: 1.68 + rank 62: 1.79 + rank 63: 1.72 + optimizer-inner-step: + rank 0: 42.66 + rank 1: 42.70 + rank 2: 42.33 + rank 3: 42.49 + rank 4: 42.32 + rank 5: 42.52 + rank 6: 42.86 + rank 7: 42.89 + rank 8: 42.66 + rank 9: 42.63 + rank 10: 42.47 + rank 11: 42.59 + rank 12: 42.60 + rank 13: 42.83 + rank 14: 42.65 + rank 15: 42.41 + rank 16: 43.00 + rank 17: 42.69 + rank 18: 42.78 + rank 19: 42.73 + rank 20: 41.75 + rank 21: 41.79 + rank 22: 43.14 + rank 23: 42.77 + rank 24: 38.82 + rank 25: 42.87 + rank 26: 40.25 + rank 27: 44.30 + rank 28: 48.09 + rank 29: 43.30 + rank 30: 42.25 + rank 31: 43.92 + rank 32: 42.70 + rank 33: 42.60 + rank 34: 42.70 + rank 35: 49.28 + rank 36: 42.77 + rank 37: 42.73 + rank 38: 42.54 + rank 39: 42.55 + rank 40: 42.54 + rank 41: 42.71 + rank 42: 42.74 + rank 43: 42.83 + rank 44: 42.42 + rank 45: 42.96 + rank 46: 42.63 + rank 47: 42.39 + rank 48: 42.69 + rank 49: 42.53 + rank 50: 54.44 + rank 51: 42.36 + rank 52: 42.75 + rank 53: 42.56 + rank 54: 42.75 + rank 55: 42.69 + rank 56: 42.32 + rank 57: 46.49 + rank 58: 42.59 + rank 59: 42.46 + rank 60: 42.81 + rank 61: 42.46 + rank 62: 42.50 + rank 63: 42.49 + optimizer-copy-main-to-model-params: + rank 0: 4.39 + rank 1: 3.84 + rank 2: 3.72 + rank 3: 4.38 + rank 4: 3.70 + rank 5: 3.96 + rank 6: 4.03 + rank 7: 3.96 + rank 8: 3.82 + rank 9: 4.47 + rank 10: 3.85 + rank 11: 3.78 + rank 12: 3.95 + rank 13: 3.81 + rank 14: 3.99 + rank 15: 4.27 + rank 16: 4.50 + rank 17: 4.01 + rank 18: 4.12 + rank 19: 4.06 + rank 20: 4.03 + rank 21: 4.21 + rank 22: 4.61 + rank 23: 4.20 + rank 24: 3.91 + rank 25: 4.21 + rank 26: 4.03 + rank 27: 4.07 + rank 28: 4.43 + rank 29: 4.41 + rank 30: 4.39 + rank 31: 4.05 + rank 32: 4.25 + rank 33: 4.09 + rank 34: 4.05 + rank 35: 4.69 + rank 36: 4.28 + rank 37: 4.31 + rank 38: 4.46 + rank 39: 4.14 + rank 40: 3.79 + rank 41: 3.99 + rank 42: 4.01 + rank 43: 4.01 + rank 44: 4.01 + rank 45: 4.23 + rank 46: 3.82 + rank 47: 3.73 + rank 48: 4.05 + rank 49: 3.89 + rank 50: 4.98 + rank 51: 3.99 + rank 52: 4.16 + rank 53: 3.87 + rank 54: 3.88 + rank 55: 3.90 + rank 56: 3.81 + rank 57: 4.68 + rank 58: 3.90 + rank 59: 3.73 + rank 60: 3.79 + rank 61: 3.81 + rank 62: 3.96 + rank 63: 3.76 + optimizer: + rank 0: 58.10 + rank 1: 57.28 + rank 2: 56.75 + rank 3: 57.81 + rank 4: 56.68 + rank 5: 57.19 + rank 6: 57.66 + rank 7: 57.62 + rank 8: 57.15 + rank 9: 58.05 + rank 10: 56.99 + rank 11: 57.02 + rank 12: 57.30 + rank 13: 57.34 + rank 14: 57.31 + rank 15: 57.57 + rank 16: 58.66 + rank 17: 57.64 + rank 18: 57.86 + rank 19: 57.68 + rank 20: 57.55 + rank 21: 57.79 + rank 22: 58.92 + rank 23: 58.09 + rank 24: 56.97 + rank 25: 61.46 + rank 26: 58.51 + rank 27: 62.62 + rank 28: 66.95 + rank 29: 62.15 + rank 30: 61.02 + rank 31: 62.21 + rank 32: 57.61 + rank 33: 57.24 + rank 34: 57.34 + rank 35: 64.81 + rank 36: 57.77 + rank 37: 57.68 + rank 38: 57.80 + rank 39: 57.27 + rank 40: 57.01 + rank 41: 57.40 + rank 42: 57.46 + rank 43: 57.55 + rank 44: 57.11 + rank 45: 58.05 + rank 46: 57.09 + rank 47: 56.74 + rank 48: 57.40 + rank 49: 57.05 + rank 50: 70.52 + rank 51: 56.96 + rank 52: 57.64 + rank 53: 57.03 + rank 54: 57.24 + rank 55: 57.17 + rank 56: 56.74 + rank 57: 62.29 + rank 58: 57.15 + rank 59: 56.76 + rank 60: 57.18 + rank 61: 56.86 + rank 62: 57.15 + rank 63: 56.82 + interval-time: + rank 0: 0.81 + rank 1: 0.83 + rank 2: 0.76 + rank 3: 0.74 + rank 4: 0.73 + rank 5: 0.87 + rank 6: 0.77 + rank 7: 0.81 + rank 8: 0.82 + rank 9: 0.85 + rank 10: 0.84 + rank 11: 0.83 + rank 12: 0.84 + rank 13: 0.73 + rank 14: 0.83 + rank 15: 0.85 + rank 16: 0.89 + rank 17: 0.84 + rank 18: 0.86 + rank 19: 0.88 + rank 20: 0.84 + rank 21: 0.82 + rank 22: 0.74 + rank 23: 0.74 + rank 24: 1.37 + rank 25: 1.32 + rank 26: 1.17 + rank 27: 1.20 + rank 28: 1.28 + rank 29: 1.31 + rank 30: 1.26 + rank 31: 1.39 + rank 32: 0.94 + rank 33: 1.00 + rank 34: 0.96 + rank 35: 0.86 + rank 36: 0.80 + rank 37: 0.83 + rank 38: 0.82 + rank 39: 0.81 + rank 40: 0.85 + rank 41: 0.83 + rank 42: 0.83 + rank 43: 0.76 + rank 44: 0.73 + rank 45: 0.80 + rank 46: 0.83 + rank 47: 0.81 + rank 48: 0.92 + rank 49: 0.85 + rank 50: 0.85 + rank 51: 0.82 + rank 52: 0.91 + rank 53: 0.80 + rank 54: 0.82 + rank 55: 0.92 + rank 56: 0.82 + rank 57: 0.72 + rank 58: 0.72 + rank 59: 0.73 + rank 60: 0.74 + rank 61: 0.74 + rank 62: 0.73 + rank 63: 1.30 + [2026-10-14 05:07:57] iteration 1006/ 2384 | consumed samples: 32192 | elapsed time per iteration (ms): 224351.0 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 74781.1 | valid tokens/s: 74781.1 | physical tokens/s: 75939.6 | loss tokens/s: 58250.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.972449E-01 | lm-global-supervised-tokens: 1.306859E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.692392E-02 | lm-token-buffer-mib: 2.356137E+02 | global_load_balancing_loss: 1.000628E+00 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11186.487 | params rms: 0.060571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:52:35.636701 | finish at 2026-10-17 19:00:33 + [2026-10-14 05:11:41] iteration 1007/ 2384 | consumed samples: 32224 | elapsed time per iteration (ms): 223993.3 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 74900.4 | valid tokens/s: 74900.4 | physical tokens/s: 75689.1 | loss tokens/s: 56238.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.757658E-01 | lm-global-supervised-tokens: 1.259708E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.557074E-02 | lm-token-buffer-mib: 2.341348E+02 | global_load_balancing_loss: 1.000842E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11186.459 | params rms: 0.060571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:40:38.707401 | finish at 2026-10-17 18:52:20 + [2026-10-14 05:15:22] iteration 1008/ 2384 | consumed samples: 32256 | elapsed time per iteration (ms): 221207.4 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75843.7 | valid tokens/s: 75843.7 | physical tokens/s: 76372.7 | loss tokens/s: 54627.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.072842E-01 | lm-global-supervised-tokens: 1.208398E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.464093E-02 | lm-token-buffer-mib: 2.339794E+02 | global_load_balancing_loss: 1.000536E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11186.432 | params rms: 0.060571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:33:01.348724 | finish at 2026-10-17 17:48:24 + [2026-10-14 05:19:04] iteration 1009/ 2384 | consumed samples: 32288 | elapsed time per iteration (ms): 221542.3 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75729.1 | valid tokens/s: 75729.1 | physical tokens/s: 76669.5 | loss tokens/s: 54383.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.980495E-01 | lm-global-supervised-tokens: 1.204817E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.841899E-02 | lm-token-buffer-mib: 2.355093E+02 | global_load_balancing_loss: 1.000445E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11186.403 | params rms: 0.060571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:37:00.655596 | finish at 2026-10-17 17:56:04 + [2026-10-14 05:23:07] iteration 1010/ 2384 | consumed samples: 32320 | elapsed time per iteration (ms): 242793.7 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 69100.6 | valid tokens/s: 69100.6 | physical tokens/s: 69687.2 | loss tokens/s: 48995.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.907034E-01 | lm-global-supervised-tokens: 1.189568E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.248956E-02 | lm-token-buffer-mib: 2.343055E+02 | global_load_balancing_loss: 1.000151E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11186.372 | params rms: 0.060571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:39:58.506755 | finish at 2026-10-18 02:03:05 + [2026-10-14 05:26:50] iteration 1011/ 2384 | consumed samples: 32352 | elapsed time per iteration (ms): 223350.3 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 75116.1 | valid tokens/s: 75116.1 | physical tokens/s: 76447.2 | loss tokens/s: 56026.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.668032E-01 | lm-global-supervised-tokens: 1.251353E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.451439E-02 | lm-token-buffer-mib: 2.334359E+02 | global_load_balancing_loss: 1.000672E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11186.340 | params rms: 0.060570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:10:59.992445 | finish at 2026-10-17 18:37:50 + [2026-10-14 05:30:33] iteration 1012/ 2384 | consumed samples: 32384 | elapsed time per iteration (ms): 223130.4 | throughput per GPU (TFLOP/s/GPU): 28.1 | MFU 9.00% | FLOPs estimator: workload-aware | logical tokens/s: 75189.9 | valid tokens/s: 75189.9 | physical tokens/s: 75534.6 | loss tokens/s: 58517.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.767725E-01 | lm-global-supervised-tokens: 1.305692E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.547235E-02 | lm-token-buffer-mib: 2.337834E+02 | global_load_balancing_loss: 1.000759E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11186.310 | params rms: 0.060570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:02:14.975779 | finish at 2026-10-17 18:32:48 + [2026-10-14 05:34:20] iteration 1013/ 2384 | consumed samples: 32416 | elapsed time per iteration (ms): 226987.0 | throughput per GPU (TFLOP/s/GPU): 27.2 | MFU 8.72% | FLOPs estimator: workload-aware | logical tokens/s: 73912.5 | valid tokens/s: 73912.5 | physical tokens/s: 74480.0 | loss tokens/s: 56287.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.836886E-01 | lm-global-supervised-tokens: 1.277663E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.288650E-02 | lm-token-buffer-mib: 2.349276E+02 | global_load_balancing_loss: 1.001836E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11186.279 | params rms: 0.060570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:26:39.181561 | finish at 2026-10-17 20:00:59 + [2026-10-14 05:38:05] iteration 1014/ 2384 | consumed samples: 32448 | elapsed time per iteration (ms): 225407.4 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 74430.4 | valid tokens/s: 74430.4 | physical tokens/s: 75112.9 | loss tokens/s: 58060.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.529981E-01 | lm-global-supervised-tokens: 1.308727E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.617487E-02 | lm-token-buffer-mib: 2.345749E+02 | global_load_balancing_loss: 9.998050E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11186.248 | params rms: 0.060570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:46:48.098004 | finish at 2026-10-17 19:24:53 + [2026-10-14 05:41:52] iteration 1015/ 2384 | consumed samples: 32480 | elapsed time per iteration (ms): 226993.0 | throughput per GPU (TFLOP/s/GPU): 25.3 | MFU 8.12% | FLOPs estimator: workload-aware | logical tokens/s: 73910.6 | valid tokens/s: 73910.6 | physical tokens/s: 74617.0 | loss tokens/s: 53680.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.015073E-01 | lm-global-supervised-tokens: 1.218516E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.483395E-02 | lm-token-buffer-mib: 2.344784E+02 | global_load_balancing_loss: 9.995416E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11186.216 | params rms: 0.060570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:19:13.435646 | finish at 2026-10-17 20:01:06 + [2026-10-14 05:45:32] iteration 1016/ 2384 | consumed samples: 32512 | elapsed time per iteration (ms): 219917.7 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 76288.5 | valid tokens/s: 76288.5 | physical tokens/s: 76586.0 | loss tokens/s: 59491.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.909363E-01 | lm-global-supervised-tokens: 1.308332E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.907593E-02 | lm-token-buffer-mib: 2.326350E+02 | global_load_balancing_loss: 9.999465E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11186.185 | params rms: 0.060570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:34:07.360834 | finish at 2026-10-17 17:19:40 + [2026-10-14 05:49:16] iteration 1017/ 2384 | consumed samples: 32544 | elapsed time per iteration (ms): 223570.2 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75042.2 | valid tokens/s: 75042.2 | physical tokens/s: 75437.0 | loss tokens/s: 54259.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.543748E-01 | lm-global-supervised-tokens: 1.213073E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.939351E-02 | lm-token-buffer-mib: 2.340385E+02 | global_load_balancing_loss: 1.001353E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11186.150 | params rms: 0.060569 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:53:40.414792 | finish at 2026-10-17 18:42:56 + [2026-10-14 05:53:02] iteration 1018/ 2384 | consumed samples: 32576 | elapsed time per iteration (ms): 226237.4 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 74157.4 | valid tokens/s: 74157.4 | physical tokens/s: 75467.6 | loss tokens/s: 58824.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.745827E-01 | lm-global-supervised-tokens: 1.330831E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.353205E-02 | lm-token-buffer-mib: 2.389044E+02 | global_load_balancing_loss: 1.000632E+00 | loss scale: 1.0 | grad norm: 0.233 | params norm: 11186.111 | params rms: 0.060569 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:50:40.282287 | finish at 2026-10-17 19:43:42 + [2026-10-14 05:56:46] iteration 1019/ 2384 | consumed samples: 32608 | elapsed time per iteration (ms): 224371.8 | throughput per GPU (TFLOP/s/GPU): 26.8 | MFU 8.60% | FLOPs estimator: workload-aware | logical tokens/s: 74774.1 | valid tokens/s: 74774.1 | physical tokens/s: 75723.7 | loss tokens/s: 57561.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.982395E-01 | lm-global-supervised-tokens: 1.291516E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.650662E-02 | lm-token-buffer-mib: 2.331957E+02 | global_load_balancing_loss: 1.000853E+00 | loss scale: 1.0 | grad norm: 0.221 | params norm: 11186.069 | params rms: 0.060569 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:04:27.503997 | finish at 2026-10-17 19:01:14 + [2026-10-14 06:00:52] iteration 1020/ 2384 | consumed samples: 32640 | elapsed time per iteration (ms): 245541.4 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 68327.3 | valid tokens/s: 68327.3 | physical tokens/s: 68826.9 | loss tokens/s: 51961.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.972911E-01 | lm-global-supervised-tokens: 1.275869E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.665528E-02 | lm-token-buffer-mib: 2.340674E+02 | global_load_balancing_loss: 1.000609E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11186.029 | params rms: 0.060569 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:01:58.407101 | finish at 2026-10-18 03:02:50 + [2026-10-14 06:04:31] iteration 1021/ 2384 | consumed samples: 32672 | elapsed time per iteration (ms): 219376.8 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 76476.6 | valid tokens/s: 76476.6 | physical tokens/s: 77091.1 | loss tokens/s: 59351.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.911457E-01 | lm-global-supervised-tokens: 1.302039E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.401810E-02 | lm-token-buffer-mib: 2.341765E+02 | global_load_balancing_loss: 1.001093E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11185.992 | params rms: 0.060569 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:03:30.548260 | finish at 2026-10-17 17:08:02 + [2026-10-14 06:08:13] iteration 1022/ 2384 | consumed samples: 32704 | elapsed time per iteration (ms): 222045.3 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.48% | FLOPs estimator: workload-aware | logical tokens/s: 75557.5 | valid tokens/s: 75557.5 | physical tokens/s: 75995.4 | loss tokens/s: 61945.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.143731E-01 | lm-global-supervised-tokens: 1.375476E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.345806E-02 | lm-token-buffer-mib: 2.334195E+02 | global_load_balancing_loss: 1.000690E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11185.955 | params rms: 0.060568 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:00:25.722314 | finish at 2026-10-17 18:08:39 + [2026-10-14 06:11:56] iteration 1023/ 2384 | consumed samples: 32736 | elapsed time per iteration (ms): 222358.8 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 75451.0 | valid tokens/s: 75451.0 | physical tokens/s: 76689.3 | loss tokens/s: 57142.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.559729E-01 | lm-global-supervised-tokens: 1.270609E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.727331E-02 | lm-token-buffer-mib: 2.342418E+02 | global_load_balancing_loss: 1.001808E+00 | loss scale: 1.0 | grad norm: 0.500 | params norm: 11185.912 | params rms: 0.060568 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:03:50.355266 | finish at 2026-10-17 18:15:46 + [2026-10-14 06:15:37] iteration 1024/ 2384 | consumed samples: 32768 | elapsed time per iteration (ms): 221470.2 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75753.8 | valid tokens/s: 75753.8 | physical tokens/s: 76484.7 | loss tokens/s: 54804.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.801921E-01 | lm-global-supervised-tokens: 1.213748E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.048733E-02 | lm-token-buffer-mib: 2.357825E+02 | global_load_balancing_loss: 1.000778E+00 | loss scale: 1.0 | grad norm: 0.291 | params norm: 11185.869 | params rms: 0.060568 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:39:59.471760 | finish at 2026-10-17 17:55:37 + [2026-10-14 06:19:21] iteration 1025/ 2384 | consumed samples: 32800 | elapsed time per iteration (ms): 223402.4 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 75098.5 | valid tokens/s: 75098.5 | physical tokens/s: 75795.1 | loss tokens/s: 56058.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.238561E-01 | lm-global-supervised-tokens: 1.252365E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.339525E-02 | lm-token-buffer-mib: 2.351872E+02 | global_load_balancing_loss: 9.997215E-01 | loss scale: 1.0 | grad norm: 0.227 | params norm: 11185.827 | params rms: 0.060568 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:20:03.796821 | finish at 2026-10-17 18:39:24 + [2026-10-14 06:23:08] iteration 1026/ 2384 | consumed samples: 32832 | elapsed time per iteration (ms): 226942.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 73927.2 | valid tokens/s: 73927.2 | physical tokens/s: 75348.9 | loss tokens/s: 55100.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.150191E-01 | lm-global-supervised-tokens: 1.250456E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.310108E-02 | lm-token-buffer-mib: 2.389412E+02 | global_load_balancing_loss: 9.999090E-01 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11185.786 | params rms: 0.060567 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:36:27.316176 | finish at 2026-10-17 19:59:35 + [2026-10-14 06:26:53] iteration 1027/ 2384 | consumed samples: 32864 | elapsed time per iteration (ms): 225329.9 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 74456.2 | valid tokens/s: 74456.2 | physical tokens/s: 75734.9 | loss tokens/s: 56109.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.240507E-01 | lm-global-supervised-tokens: 1.264306E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.510544E-02 | lm-token-buffer-mib: 2.355058E+02 | global_load_balancing_loss: 1.000838E+00 | loss scale: 1.0 | grad norm: 0.245 | params norm: 11185.747 | params rms: 0.060567 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:56:12.695365 | finish at 2026-10-17 19:23:06 + [2026-10-14 06:30:36] iteration 1028/ 2384 | consumed samples: 32896 | elapsed time per iteration (ms): 223256.3 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.75% | FLOPs estimator: workload-aware | logical tokens/s: 75147.7 | valid tokens/s: 75147.7 | physical tokens/s: 75409.8 | loss tokens/s: 53459.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.143726E-01 | lm-global-supervised-tokens: 1.193519E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.522509E-02 | lm-token-buffer-mib: 2.330376E+02 | global_load_balancing_loss: 9.988275E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11185.703 | params rms: 0.060567 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:05:35.548582 | finish at 2026-10-17 18:36:12 + [2026-10-14 06:34:19] iteration 1029/ 2384 | consumed samples: 32928 | elapsed time per iteration (ms): 223021.9 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 75226.7 | valid tokens/s: 75226.7 | physical tokens/s: 75820.8 | loss tokens/s: 57398.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.647581E-01 | lm-global-supervised-tokens: 1.280114E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.915837E-02 | lm-token-buffer-mib: 2.343493E+02 | global_load_balancing_loss: 9.994837E-01 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11185.657 | params rms: 0.060567 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:56:34.716414 | finish at 2026-10-17 18:30:54 + [2026-10-14 06:38:25] iteration 1030/ 2384 | consumed samples: 32960 | elapsed time per iteration (ms): 245956.2 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 68212.1 | valid tokens/s: 68212.1 | physical tokens/s: 69152.3 | loss tokens/s: 51602.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.078554E-01 | lm-global-supervised-tokens: 1.269193E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.650588E-02 | lm-token-buffer-mib: 2.343695E+02 | global_load_balancing_loss: 1.000763E+00 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11185.612 | params rms: 0.060566 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:30:24.675597 | finish at 2026-10-18 03:08:50 + [2026-10-14 06:42:08] iteration 1031/ 2384 | consumed samples: 32992 | elapsed time per iteration (ms): 222666.9 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.47% | FLOPs estimator: workload-aware | logical tokens/s: 75346.6 | valid tokens/s: 75346.6 | physical tokens/s: 76478.0 | loss tokens/s: 59216.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.369941E-01 | lm-global-supervised-tokens: 1.318552E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.695010E-02 | lm-token-buffer-mib: 2.348992E+02 | global_load_balancing_loss: 1.000023E+00 | loss scale: 1.0 | grad norm: 0.243 | params norm: 11185.567 | params rms: 0.060566 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:41:08.267849 | finish at 2026-10-17 18:23:16 + [2026-10-14 06:45:52] iteration 1032/ 2384 | consumed samples: 33024 | elapsed time per iteration (ms): 224565.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 74709.8 | valid tokens/s: 74709.8 | physical tokens/s: 75705.0 | loss tokens/s: 59170.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.732852E-01 | lm-global-supervised-tokens: 1.328760E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.674205E-02 | lm-token-buffer-mib: 2.361547E+02 | global_load_balancing_loss: 1.000647E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11185.523 | params rms: 0.060566 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:20:12.011271 | finish at 2026-10-17 19:06:04 + [2026-10-14 06:49:34] iteration 1033/ 2384 | consumed samples: 33056 | elapsed time per iteration (ms): 221096.6 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.93% | FLOPs estimator: workload-aware | logical tokens/s: 75881.6 | valid tokens/s: 75881.6 | physical tokens/s: 76508.5 | loss tokens/s: 56206.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.467679E-01 | lm-global-supervised-tokens: 1.242716E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.598163E-02 | lm-token-buffer-mib: 2.352909E+02 | global_load_balancing_loss: 1.001903E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11185.475 | params rms: 0.060566 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:58:21.518593 | finish at 2026-10-17 17:47:55 + [2026-10-14 06:53:15] iteration 1034/ 2384 | consumed samples: 33088 | elapsed time per iteration (ms): 221860.5 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 75620.5 | valid tokens/s: 75620.5 | physical tokens/s: 76196.0 | loss tokens/s: 54768.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.640628E-01 | lm-global-supervised-tokens: 1.215085E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.189216E-02 | lm-token-buffer-mib: 2.336559E+02 | global_load_balancing_loss: 1.001376E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11185.427 | params rms: 0.060565 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:11:51.720192 | finish at 2026-10-17 18:05:07 + [2026-10-14 06:56:58] iteration 1035/ 2384 | consumed samples: 33120 | elapsed time per iteration (ms): 222878.1 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 75275.3 | valid tokens/s: 75275.3 | physical tokens/s: 76032.5 | loss tokens/s: 58100.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.620819E-01 | lm-global-supervised-tokens: 1.294934E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.673533E-02 | lm-token-buffer-mib: 2.342624E+02 | global_load_balancing_loss: 1.000859E+00 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11185.382 | params rms: 0.060565 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:31:02.536206 | finish at 2026-10-17 18:28:01 + [2026-10-14 07:00:40] iteration 1036/ 2384 | consumed samples: 33152 | elapsed time per iteration (ms): 221812.2 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 75636.9 | valid tokens/s: 75636.9 | physical tokens/s: 76816.5 | loss tokens/s: 61639.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.422580E-01 | lm-global-supervised-tokens: 1.367234E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.731673E-02 | lm-token-buffer-mib: 2.345625E+02 | global_load_balancing_loss: 1.000426E+00 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11185.341 | params rms: 0.060565 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:03:22.903865 | finish at 2026-10-17 18:04:03 + [2026-10-14 07:04:24] iteration 1037/ 2384 | consumed samples: 33184 | elapsed time per iteration (ms): 223568.0 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 75042.9 | valid tokens/s: 75042.9 | physical tokens/s: 76182.4 | loss tokens/s: 55379.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.065056E-01 | lm-global-supervised-tokens: 1.238099E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.362455E-02 | lm-token-buffer-mib: 2.353853E+02 | global_load_balancing_loss: 9.999490E-01 | loss scale: 1.0 | grad norm: 0.199 | params norm: 11185.300 | params rms: 0.060565 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:39:06.039583 | finish at 2026-10-17 18:43:30 + [2026-10-14 07:08:04] iteration 1038/ 2384 | consumed samples: 33216 | elapsed time per iteration (ms): 220652.0 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 76034.5 | valid tokens/s: 76034.5 | physical tokens/s: 76963.5 | loss tokens/s: 58249.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.636015E-01 | lm-global-supervised-tokens: 1.285295E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.771854E-02 | lm-token-buffer-mib: 2.346358E+02 | global_load_balancing_loss: 9.991902E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11185.259 | params rms: 0.060565 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:29:57.603486 | finish at 2026-10-17 17:38:02 + [2026-10-14 07:11:46] iteration 1039/ 2384 | consumed samples: 33248 | elapsed time per iteration (ms): 221470.9 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 75753.6 | valid tokens/s: 75753.6 | physical tokens/s: 76422.3 | loss tokens/s: 56748.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.483727E-01 | lm-global-supervised-tokens: 1.256809E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.243236E-02 | lm-token-buffer-mib: 2.343089E+02 | global_load_balancing_loss: 1.000578E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11185.218 | params rms: 0.060564 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:44:38.394570 | finish at 2026-10-17 17:56:24 + [2026-10-14 07:15:51] iteration 1040/ 2384 | consumed samples: 33280 | elapsed time per iteration (ms): 245490.5 | throughput per GPU (TFLOP/s/GPU): 17.5 | MFU 5.62% | FLOPs estimator: workload-aware | logical tokens/s: 68341.6 | valid tokens/s: 68341.6 | physical tokens/s: 69446.2 | loss tokens/s: 52644.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.842547E-01 | lm-global-supervised-tokens: 1.292366E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.079705E-02 | lm-token-buffer-mib: 2.351038E+02 | global_load_balancing_loss: 9.992430E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11185.176 | params rms: 0.060564 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:38:59.284515 | finish at 2026-10-18 02:54:51 +times across ranks (ms): + save-checkpoint: + rank 0: 26818.75 + rank 1: 26818.76 + rank 2: 26818.84 + rank 3: 26818.78 + rank 4: 26818.75 + rank 5: 26818.77 + rank 6: 26818.76 + rank 7: 26818.76 + rank 8: 26817.64 + rank 9: 26817.65 + rank 10: 26817.70 + rank 11: 26817.64 + rank 12: 26817.63 + rank 13: 26817.63 + rank 14: 26817.63 + rank 15: 26817.48 + rank 16: 26818.80 + rank 17: 26818.76 + rank 18: 26818.79 + rank 19: 26818.76 + rank 20: 26818.75 + rank 21: 26818.77 + rank 22: 26818.82 + rank 23: 26818.73 + rank 24: 26818.61 + rank 25: 26818.64 + rank 26: 26818.58 + rank 27: 26818.68 + rank 28: 26818.57 + rank 29: 26818.60 + rank 30: 26818.70 + rank 31: 26818.71 + rank 32: 26818.07 + rank 33: 26818.05 + rank 34: 26818.11 + rank 35: 26818.07 + rank 36: 26818.02 + rank 37: 26818.09 + rank 38: 26818.05 + rank 39: 26818.19 + rank 40: 26819.45 + rank 41: 26819.44 + rank 42: 26819.44 + rank 43: 26819.45 + rank 44: 26819.42 + rank 45: 26819.46 + rank 46: 26819.43 + rank 47: 26819.42 + rank 48: 26818.82 + rank 49: 26818.86 + rank 50: 26818.85 + rank 51: 26818.82 + rank 52: 26818.82 + rank 53: 26818.86 + rank 54: 26818.82 + rank 55: 26818.84 + rank 56: 26818.65 + rank 57: 26818.69 + rank 58: 26818.70 + rank 59: 26818.71 + rank 60: 26818.70 + rank 61: 26818.72 + rank 62: 26818.67 + rank 63: 26818.66 + [2026-10-14 07:20:23] iteration 1041/ 2384 | consumed samples: 33312 | elapsed time per iteration (ms): 245377.9 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.28% | FLOPs estimator: workload-aware | logical tokens/s: 68372.7 | valid tokens/s: 68372.7 | physical tokens/s: 69333.8 | loss tokens/s: 50535.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.171594E-01 | lm-global-supervised-tokens: 1.240026E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.897097E-02 | lm-token-buffer-mib: 2.357247E+02 | global_load_balancing_loss: 9.994258E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11185.135 | params rms: 0.060564 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:32:22.581344 | finish at 2026-10-18 02:52:46 + [2026-10-14 07:24:10] iteration 1042/ 2384 | consumed samples: 33344 | elapsed time per iteration (ms): 226570.5 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 74048.4 | valid tokens/s: 74048.4 | physical tokens/s: 74518.0 | loss tokens/s: 56024.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.898151E-01 | lm-global-supervised-tokens: 1.269338E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.359328E-02 | lm-token-buffer-mib: 2.335364E+02 | global_load_balancing_loss: 1.001005E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11185.092 | params rms: 0.060564 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:27:37.656616 | finish at 2026-10-17 19:51:48 + [2026-10-14 07:27:53] iteration 1043/ 2384 | consumed samples: 33376 | elapsed time per iteration (ms): 222525.4 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.09% | FLOPs estimator: workload-aware | logical tokens/s: 75394.5 | valid tokens/s: 75394.5 | physical tokens/s: 75806.7 | loss tokens/s: 56960.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.219561E-01 | lm-global-supervised-tokens: 1.267517E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.530527E-02 | lm-token-buffer-mib: 2.334686E+02 | global_load_balancing_loss: 1.000007E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11185.048 | params rms: 0.060563 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:53:26.577283 | finish at 2026-10-17 18:21:19 + [2026-10-14 07:31:32] iteration 1044/ 2384 | consumed samples: 33408 | elapsed time per iteration (ms): 219581.8 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 76405.1 | valid tokens/s: 76405.1 | physical tokens/s: 77217.6 | loss tokens/s: 55580.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.021168E-01 | lm-global-supervised-tokens: 1.220436E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.478486E-02 | lm-token-buffer-mib: 2.340067E+02 | global_load_balancing_loss: 1.001184E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11185.006 | params rms: 0.060563 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:43:59.651594 | finish at 2026-10-17 17:15:32 + [2026-10-14 07:35:14] iteration 1045/ 2384 | consumed samples: 33440 | elapsed time per iteration (ms): 221581.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75715.7 | valid tokens/s: 75715.7 | physical tokens/s: 76726.8 | loss tokens/s: 58197.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.139081E-01 | lm-global-supervised-tokens: 1.289556E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.582325E-02 | lm-token-buffer-mib: 2.346768E+02 | global_load_balancing_loss: 1.001889E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11184.967 | params rms: 0.060563 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:24:57.153219 | finish at 2026-10-17 18:00:11 + [2026-10-14 07:38:54] iteration 1046/ 2384 | consumed samples: 33472 | elapsed time per iteration (ms): 220359.1 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 76135.8 | valid tokens/s: 76135.8 | physical tokens/s: 76932.5 | loss tokens/s: 58664.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.697501E-01 | lm-global-supervised-tokens: 1.292723E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.725212E-02 | lm-token-buffer-mib: 2.341048E+02 | global_load_balancing_loss: 1.001663E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11184.933 | params rms: 0.060563 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:54:00.492208 | finish at 2026-10-17 17:32:55 + [2026-10-14 07:42:35] iteration 1047/ 2384 | consumed samples: 33504 | elapsed time per iteration (ms): 220631.5 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 76041.7 | valid tokens/s: 76041.7 | physical tokens/s: 76466.5 | loss tokens/s: 52119.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.160988E-01 | lm-global-supervised-tokens: 1.149926E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.609233E-02 | lm-token-buffer-mib: 2.325928E+02 | global_load_balancing_loss: 1.001089E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11184.896 | params rms: 0.060563 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:56:24.359497 | finish at 2026-10-17 17:38:59 + [2026-10-14 07:46:16] iteration 1048/ 2384 | consumed samples: 33536 | elapsed time per iteration (ms): 221712.3 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 75671.0 | valid tokens/s: 75671.0 | physical tokens/s: 76186.4 | loss tokens/s: 56828.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.861825E-01 | lm-global-supervised-tokens: 1.259963E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.250518E-02 | lm-token-buffer-mib: 2.345108E+02 | global_load_balancing_loss: 9.996719E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11184.858 | params rms: 0.060562 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:16:47.581919 | finish at 2026-10-17 18:03:04 + [2026-10-14 07:49:59] iteration 1049/ 2384 | consumed samples: 33568 | elapsed time per iteration (ms): 222675.8 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75343.5 | valid tokens/s: 75343.5 | physical tokens/s: 76066.0 | loss tokens/s: 60211.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.965376E-01 | lm-global-supervised-tokens: 1.340758E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.681384E-02 | lm-token-buffer-mib: 2.336336E+02 | global_load_balancing_loss: 9.990115E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11184.815 | params rms: 0.060562 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:34:32.167332 | finish at 2026-10-17 18:24:31 + [2026-10-14 07:54:03] iteration 1050/ 2384 | consumed samples: 33600 | elapsed time per iteration (ms): 244391.9 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 68648.7 | valid tokens/s: 68648.7 | physical tokens/s: 69805.3 | loss tokens/s: 52590.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.664661E-01 | lm-global-supervised-tokens: 1.285262E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.015629E-02 | lm-token-buffer-mib: 2.336388E+02 | global_load_balancing_loss: 9.989833E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11184.774 | params rms: 0.060562 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:33:38.787686 | finish at 2026-10-18 02:27:42 + [2026-10-14 07:57:50] iteration 1051/ 2384 | consumed samples: 33632 | elapsed time per iteration (ms): 226088.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 74206.5 | valid tokens/s: 74206.5 | physical tokens/s: 75331.8 | loss tokens/s: 58939.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.369125E-01 | lm-global-supervised-tokens: 1.332561E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.815994E-02 | lm-token-buffer-mib: 2.361445E+02 | global_load_balancing_loss: 9.972283E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11184.732 | params rms: 0.060562 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:42:55.407050 | finish at 2026-10-17 19:40:45 + [2026-10-14 08:01:31] iteration 1052/ 2384 | consumed samples: 33664 | elapsed time per iteration (ms): 221582.7 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 75715.1 | valid tokens/s: 75715.1 | physical tokens/s: 76221.1 | loss tokens/s: 56338.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.587636E-01 | lm-global-supervised-tokens: 1.248369E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.501899E-02 | lm-token-buffer-mib: 2.341118E+02 | global_load_balancing_loss: 1.000156E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11184.691 | params rms: 0.060561 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:59:08.158324 | finish at 2026-10-17 18:00:39 + [2026-10-14 08:05:13] iteration 1053/ 2384 | consumed samples: 33696 | elapsed time per iteration (ms): 221760.8 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75654.5 | valid tokens/s: 75654.5 | physical tokens/s: 76220.2 | loss tokens/s: 57296.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.128121E-01 | lm-global-supervised-tokens: 1.270605E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.563671E-02 | lm-token-buffer-mib: 2.335134E+02 | global_load_balancing_loss: 1.001859E+00 | loss scale: 1.0 | grad norm: 0.257 | params norm: 11184.650 | params rms: 0.060561 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:59:23.620204 | finish at 2026-10-17 18:04:37 + [2026-10-14 08:08:57] iteration 1054/ 2384 | consumed samples: 33728 | elapsed time per iteration (ms): 223807.0 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 74962.7 | valid tokens/s: 74962.7 | physical tokens/s: 75588.9 | loss tokens/s: 57731.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.919155E-01 | lm-global-supervised-tokens: 1.292081E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.951456E-02 | lm-token-buffer-mib: 2.331489E+02 | global_load_balancing_loss: 1.003019E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11184.610 | params rms: 0.060561 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:41:03.272796 | finish at 2026-10-17 18:50:00 + [2026-10-14 08:12:39] iteration 1055/ 2384 | consumed samples: 33760 | elapsed time per iteration (ms): 222174.6 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 75513.6 | valid tokens/s: 75513.6 | physical tokens/s: 76080.5 | loss tokens/s: 58565.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.035038E-01 | lm-global-supervised-tokens: 1.301172E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.636132E-02 | lm-token-buffer-mib: 2.327606E+02 | global_load_balancing_loss: 1.002303E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11184.571 | params rms: 0.060561 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:01:10.099966 | finish at 2026-10-17 18:13:49 + [2026-10-14 08:16:24] iteration 1056/ 2384 | consumed samples: 33792 | elapsed time per iteration (ms): 224880.0 | throughput per GPU (TFLOP/s/GPU): 18.1 | MFU 5.79% | FLOPs estimator: workload-aware | logical tokens/s: 74605.0 | valid tokens/s: 74605.0 | physical tokens/s: 75894.4 | loss tokens/s: 57813.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.624006E-01 | lm-global-supervised-tokens: 1.300106E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.838530E-02 | lm-token-buffer-mib: 2.365938E+02 | global_load_balancing_loss: 1.001307E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11184.530 | params rms: 0.060561 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:57:20.624638 | finish at 2026-10-17 19:13:44 + [2026-10-14 08:20:06] iteration 1057/ 2384 | consumed samples: 33824 | elapsed time per iteration (ms): 222025.5 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 75564.2 | valid tokens/s: 75564.2 | physical tokens/s: 76438.6 | loss tokens/s: 54850.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.084721E-01 | lm-global-supervised-tokens: 1.217818E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.798930E-02 | lm-token-buffer-mib: 2.353470E+02 | global_load_balancing_loss: 9.995781E-01 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11184.492 | params rms: 0.060560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:50:27.865471 | finish at 2026-10-17 18:10:34 + [2026-10-14 08:23:48] iteration 1058/ 2384 | consumed samples: 33856 | elapsed time per iteration (ms): 221973.3 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.14% | FLOPs estimator: workload-aware | logical tokens/s: 75582.1 | valid tokens/s: 75582.1 | physical tokens/s: 76034.5 | loss tokens/s: 55995.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.098938E-01 | lm-global-supervised-tokens: 1.242958E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.493310E-02 | lm-token-buffer-mib: 2.328396E+02 | global_load_balancing_loss: 9.993728E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11184.456 | params rms: 0.060560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:45:36.530332 | finish at 2026-10-17 18:09:24 + [2026-10-14 08:27:29] iteration 1059/ 2384 | consumed samples: 33888 | elapsed time per iteration (ms): 221410.2 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75774.2 | valid tokens/s: 75774.2 | physical tokens/s: 76593.8 | loss tokens/s: 57244.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.301282E-01 | lm-global-supervised-tokens: 1.267454E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.078365E-02 | lm-token-buffer-mib: 2.340298E+02 | global_load_balancing_loss: 9.986334E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11184.421 | params rms: 0.060560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:29:28.499362 | finish at 2026-10-17 17:56:58 + [2026-10-14 08:31:30] iteration 1060/ 2384 | consumed samples: 33920 | elapsed time per iteration (ms): 240783.7 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 69677.5 | valid tokens/s: 69677.5 | physical tokens/s: 70333.0 | loss tokens/s: 54930.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.419765E-01 | lm-global-supervised-tokens: 1.322636E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.271355E-02 | lm-token-buffer-mib: 2.338602E+02 | global_load_balancing_loss: 9.987599E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11184.387 | params rms: 0.060560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:33:17.595742 | finish at 2026-10-18 01:04:48 + [2026-10-14 08:35:14] iteration 1061/ 2384 | consumed samples: 33952 | elapsed time per iteration (ms): 223618.9 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75025.8 | valid tokens/s: 75025.8 | physical tokens/s: 75639.5 | loss tokens/s: 56391.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.287422E-01 | lm-global-supervised-tokens: 1.261022E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.613005E-02 | lm-token-buffer-mib: 2.350333E+02 | global_load_balancing_loss: 9.989091E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11184.353 | params rms: 0.060560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:10:47.799418 | finish at 2026-10-17 18:46:01 + [2026-10-14 08:38:56] iteration 1062/ 2384 | consumed samples: 33984 | elapsed time per iteration (ms): 222627.8 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75359.8 | valid tokens/s: 75359.8 | physical tokens/s: 76151.9 | loss tokens/s: 55511.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.520961E-01 | lm-global-supervised-tokens: 1.235835E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.994098E-02 | lm-token-buffer-mib: 2.351330E+02 | global_load_balancing_loss: 1.000919E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11184.320 | params rms: 0.060559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:45:13.966083 | finish at 2026-10-17 18:24:10 + [2026-10-14 08:42:39] iteration 1063/ 2384 | consumed samples: 34016 | elapsed time per iteration (ms): 222960.9 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75247.2 | valid tokens/s: 75247.2 | physical tokens/s: 75900.7 | loss tokens/s: 58780.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.916618E-01 | lm-global-supervised-tokens: 1.310569E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.033174E-02 | lm-token-buffer-mib: 2.337688E+02 | global_load_balancing_loss: 1.000459E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11184.287 | params rms: 0.060559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:48:51.331668 | finish at 2026-10-17 18:31:31 + [2026-10-14 08:46:19] iteration 1064/ 2384 | consumed samples: 34048 | elapsed time per iteration (ms): 219882.6 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 76300.7 | valid tokens/s: 76300.7 | physical tokens/s: 76717.0 | loss tokens/s: 56713.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.193625E-01 | lm-global-supervised-tokens: 1.247024E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.649742E-02 | lm-token-buffer-mib: 2.335021E+02 | global_load_balancing_loss: 1.002096E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11184.250 | params rms: 0.060559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:37:24.993696 | finish at 2026-10-17 17:23:44 + [2026-10-14 08:50:59] iteration 1065/ 2384 | consumed samples: 34080 | elapsed time per iteration (ms): 280003.6 | throughput per GPU (TFLOP/s/GPU): 14.8 | MFU 4.75% | FLOPs estimator: workload-aware | logical tokens/s: 58216.6 | valid tokens/s: 58216.6 | physical tokens/s: 59197.2 | loss tokens/s: 44958.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.537838E-01 | lm-global-supervised-tokens: 1.258857E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.155370E-01 | lm-token-buffer-mib: 2.219308E+02 | global_load_balancing_loss: 1.001261E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11184.217 | params rms: 0.060559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:35:24.702331 | finish at 2026-10-18 15:26:24 + [2026-10-14 08:54:40] iteration 1066/ 2384 | consumed samples: 34112 | elapsed time per iteration (ms): 221340.7 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.07% | FLOPs estimator: workload-aware | logical tokens/s: 75798.0 | valid tokens/s: 75798.0 | physical tokens/s: 76354.2 | loss tokens/s: 56518.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.943617E-01 | lm-global-supervised-tokens: 1.250988E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.491351E-02 | lm-token-buffer-mib: 2.326044E+02 | global_load_balancing_loss: 1.000987E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11184.182 | params rms: 0.060559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:02:07.074535 | finish at 2026-10-17 17:56:48 + [2026-10-14 08:58:27] iteration 1067/ 2384 | consumed samples: 34144 | elapsed time per iteration (ms): 226481.7 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.25% | FLOPs estimator: workload-aware | logical tokens/s: 74077.5 | valid tokens/s: 74077.5 | physical tokens/s: 75334.8 | loss tokens/s: 54339.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.325812E-01 | lm-global-supervised-tokens: 1.230687E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.506657E-02 | lm-token-buffer-mib: 2.359335E+02 | global_load_balancing_loss: 1.001762E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11184.146 | params rms: 0.060559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:51:16.400082 | finish at 2026-10-17 19:49:43 + [2026-10-14 09:02:12] iteration 1068/ 2384 | consumed samples: 34176 | elapsed time per iteration (ms): 224954.9 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 74580.1 | valid tokens/s: 74580.1 | physical tokens/s: 74965.3 | loss tokens/s: 58601.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.572635E-01 | lm-global-supervised-tokens: 1.318273E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.399136E-02 | lm-token-buffer-mib: 2.326167E+02 | global_load_balancing_loss: 1.000545E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11184.113 | params rms: 0.060558 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:14:00.587879 | finish at 2026-10-17 19:16:12 + [2026-10-14 09:05:53] iteration 1069/ 2384 | consumed samples: 34208 | elapsed time per iteration (ms): 220681.1 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.72% | FLOPs estimator: workload-aware | logical tokens/s: 76024.5 | valid tokens/s: 76024.5 | physical tokens/s: 76367.8 | loss tokens/s: 54361.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.910494E-01 | lm-global-supervised-tokens: 1.199654E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.384868E-02 | lm-token-buffer-mib: 2.327961E+02 | global_load_balancing_loss: 9.996989E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11184.081 | params rms: 0.060558 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:36:35.664855 | finish at 2026-10-17 17:42:28 + [2026-10-14 09:09:55] iteration 1070/ 2384 | consumed samples: 34240 | elapsed time per iteration (ms): 242159.8 | throughput per GPU (TFLOP/s/GPU): 17.3 | MFU 5.55% | FLOPs estimator: workload-aware | logical tokens/s: 69281.6 | valid tokens/s: 69281.6 | physical tokens/s: 69821.0 | loss tokens/s: 50069.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.156121E-01 | lm-global-supervised-tokens: 1.212478E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.768966E-02 | lm-token-buffer-mib: 2.337632E+02 | global_load_balancing_loss: 9.985169E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11184.048 | params rms: 0.060558 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:23:17.964738 | finish at 2026-10-18 01:33:13 + [2026-10-14 09:13:38] iteration 1071/ 2384 | consumed samples: 34272 | elapsed time per iteration (ms): 223686.7 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.63% | FLOPs estimator: workload-aware | logical tokens/s: 75003.1 | valid tokens/s: 75003.1 | physical tokens/s: 75473.6 | loss tokens/s: 58873.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.338374E-01 | lm-global-supervised-tokens: 1.316927E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.545688E-02 | lm-token-buffer-mib: 2.336231E+02 | global_load_balancing_loss: 9.989818E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11184.016 | params rms: 0.060558 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:35:00.661030 | finish at 2026-10-17 18:48:39 + [2026-10-14 09:17:20] iteration 1072/ 2384 | consumed samples: 34304 | elapsed time per iteration (ms): 221665.0 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 75687.1 | valid tokens/s: 75687.1 | physical tokens/s: 76608.6 | loss tokens/s: 57850.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.542792E-01 | lm-global-supervised-tokens: 1.282350E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.454102E-02 | lm-token-buffer-mib: 2.356472E+02 | global_load_balancing_loss: 9.993902E-01 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11183.983 | params rms: 0.060558 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:47:04.468689 | finish at 2026-10-17 18:04:25 + [2026-10-14 09:21:02] iteration 1073/ 2384 | consumed samples: 34336 | elapsed time per iteration (ms): 221588.9 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 75713.2 | valid tokens/s: 75713.2 | physical tokens/s: 76144.2 | loss tokens/s: 57921.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.786572E-01 | lm-global-supervised-tokens: 1.283465E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.900428E-02 | lm-token-buffer-mib: 2.330591E+02 | global_load_balancing_loss: 1.000035E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11183.948 | params rms: 0.060557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:41:42.995193 | finish at 2026-10-17 18:02:45 + [2026-10-14 09:24:45] iteration 1074/ 2384 | consumed samples: 34368 | elapsed time per iteration (ms): 223123.7 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 75192.4 | valid tokens/s: 75192.4 | physical tokens/s: 76106.9 | loss tokens/s: 57700.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.437517E-01 | lm-global-supervised-tokens: 1.287444E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.253870E-02 | lm-token-buffer-mib: 2.352391E+02 | global_load_balancing_loss: 1.000300E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11183.914 | params rms: 0.060557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:11:32.107465 | finish at 2026-10-17 18:36:17 + [2026-10-14 09:28:25] iteration 1075/ 2384 | consumed samples: 34400 | elapsed time per iteration (ms): 220191.1 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 76193.8 | valid tokens/s: 76193.7 | physical tokens/s: 76628.1 | loss tokens/s: 54651.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.761000E-01 | lm-global-supervised-tokens: 1.203373E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.661140E-02 | lm-token-buffer-mib: 2.326800E+02 | global_load_balancing_loss: 1.000742E+00 | loss scale: 1.0 | grad norm: 0.467 | params norm: 11183.879 | params rms: 0.060557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:03:50.090761 | finish at 2026-10-17 17:32:15 + [2026-10-14 09:32:06] iteration 1076/ 2384 | consumed samples: 34432 | elapsed time per iteration (ms): 221114.2 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 75875.8 | valid tokens/s: 75875.7 | physical tokens/s: 76558.3 | loss tokens/s: 60238.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.163088E-01 | lm-global-supervised-tokens: 1.331961E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.812370E-02 | lm-token-buffer-mib: 2.347467E+02 | global_load_balancing_loss: 1.001008E+00 | loss scale: 1.0 | grad norm: 0.286 | params norm: 11183.844 | params rms: 0.060557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:20:17.339759 | finish at 2026-10-17 17:52:23 + [2026-10-14 09:35:54] iteration 1077/ 2384 | consumed samples: 34464 | elapsed time per iteration (ms): 228122.0 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.44% | FLOPs estimator: workload-aware | logical tokens/s: 73544.8 | valid tokens/s: 73544.8 | physical tokens/s: 74800.4 | loss tokens/s: 56559.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.506998E-01 | lm-global-supervised-tokens: 1.290248E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.305271E-02 | lm-token-buffer-mib: 2.363848E+02 | global_load_balancing_loss: 1.002874E+00 | loss scale: 1.0 | grad norm: 0.688 | params norm: 11183.812 | params rms: 0.060557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:49:15.446182 | finish at 2026-10-17 20:25:10 + [2026-10-14 09:39:41] iteration 1078/ 2384 | consumed samples: 34496 | elapsed time per iteration (ms): 226471.9 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 74080.7 | valid tokens/s: 74080.7 | physical tokens/s: 74667.8 | loss tokens/s: 57658.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.981524E-01 | lm-global-supervised-tokens: 1.305812E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.510434E-01 | lm-token-buffer-mib: 2.332577E+02 | global_load_balancing_loss: 9.997511E-01 | loss scale: 1.0 | grad norm: 0.398 | params norm: 11183.779 | params rms: 0.060557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:09:32.242532 | finish at 2026-10-17 19:49:13 + [2026-10-14 09:43:20] iteration 1079/ 2384 | consumed samples: 34528 | elapsed time per iteration (ms): 219465.5 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 76445.6 | valid tokens/s: 76445.6 | physical tokens/s: 77297.6 | loss tokens/s: 59495.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.524089E-01 | lm-global-supervised-tokens: 1.305722E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.697651E-02 | lm-token-buffer-mib: 2.338020E+02 | global_load_balancing_loss: 9.998580E-01 | loss scale: 1.0 | grad norm: 0.410 | params norm: 11183.744 | params rms: 0.060556 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:33:22.444983 | finish at 2026-10-17 17:16:43 + [2026-10-14 09:47:28] iteration 1080/ 2384 | consumed samples: 34560 | elapsed time per iteration (ms): 247782.1 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 67709.4 | valid tokens/s: 67709.4 | physical tokens/s: 68489.3 | loss tokens/s: 54348.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.551488E-01 | lm-global-supervised-tokens: 1.346666E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.229120E-02 | lm-token-buffer-mib: 2.331079E+02 | global_load_balancing_loss: 9.998855E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11183.711 | params rms: 0.060556 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:45:07.883533 | finish at 2026-10-18 03:32:36 +times across ranks (ms): + save-checkpoint: + rank 0: 24306.90 + rank 1: 24306.90 + rank 2: 24306.94 + rank 3: 24306.91 + rank 4: 24306.90 + rank 5: 24306.92 + rank 6: 24306.89 + rank 7: 24306.90 + rank 8: 24307.10 + rank 9: 24307.09 + rank 10: 24307.11 + rank 11: 24307.07 + rank 12: 24307.08 + rank 13: 24307.06 + rank 14: 24307.06 + rank 15: 24307.09 + rank 16: 24306.01 + rank 17: 24305.91 + rank 18: 24305.97 + rank 19: 24305.92 + rank 20: 24305.91 + rank 21: 24305.93 + rank 22: 24305.92 + rank 23: 24305.93 + rank 24: 24305.15 + rank 25: 24305.13 + rank 26: 24305.14 + rank 27: 24305.15 + rank 28: 24305.11 + rank 29: 24305.11 + rank 30: 24305.11 + rank 31: 24305.10 + rank 32: 24307.39 + rank 33: 24307.38 + rank 34: 24307.35 + rank 35: 24307.37 + rank 36: 24307.33 + rank 37: 24307.41 + rank 38: 24307.35 + rank 39: 24307.34 + rank 40: 24307.09 + rank 41: 24307.05 + rank 42: 24307.08 + rank 43: 24307.08 + rank 44: 24307.03 + rank 45: 24307.04 + rank 46: 24307.03 + rank 47: 24307.02 + rank 48: 24309.29 + rank 49: 24309.33 + rank 50: 24309.32 + rank 51: 24309.29 + rank 52: 24309.30 + rank 53: 24309.30 + rank 54: 24309.29 + rank 55: 24309.29 + rank 56: 24307.28 + rank 57: 24307.33 + rank 58: 24307.28 + rank 59: 24307.28 + rank 60: 24307.29 + rank 61: 24307.26 + rank 62: 24307.25 + rank 63: 24307.25 + [2026-10-14 09:51:32] iteration 1081/ 2384 | consumed samples: 34592 | elapsed time per iteration (ms): 220142.5 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 76210.6 | valid tokens/s: 76210.6 | physical tokens/s: 76927.3 | loss tokens/s: 57008.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.051285E-01 | lm-global-supervised-tokens: 1.255004E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.898089E-02 | lm-token-buffer-mib: 2.354552E+02 | global_load_balancing_loss: 9.997454E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11183.677 | params rms: 0.060556 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:40:45.664974 | finish at 2026-10-17 17:32:18 + [2026-10-14 09:55:50] iteration 1082/ 2384 | consumed samples: 34624 | elapsed time per iteration (ms): 257653.5 | throughput per GPU (TFLOP/s/GPU): 16.9 | MFU 5.42% | FLOPs estimator: workload-aware | logical tokens/s: 65115.3 | valid tokens/s: 65115.3 | physical tokens/s: 65542.9 | loss tokens/s: 52227.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.117937E-01 | lm-global-supervised-tokens: 1.345651E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.300512E-02 | lm-token-buffer-mib: 2.341978E+02 | global_load_balancing_loss: 9.999726E-01 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11183.640 | params rms: 0.060556 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:11:04.914222 | finish at 2026-10-18 07:06:55 + [2026-10-14 09:59:30] iteration 1083/ 2384 | consumed samples: 34656 | elapsed time per iteration (ms): 220038.0 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.28% | FLOPs estimator: workload-aware | logical tokens/s: 76246.8 | valid tokens/s: 76246.8 | physical tokens/s: 76657.6 | loss tokens/s: 53564.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.397920E-01 | lm-global-supervised-tokens: 1.178612E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.383863E-02 | lm-token-buffer-mib: 2.329685E+02 | global_load_balancing_loss: 1.000994E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11183.605 | params rms: 0.060556 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:31:09.474741 | finish at 2026-10-17 17:30:40 + [2026-10-14 10:03:15] iteration 1084/ 2384 | consumed samples: 34688 | elapsed time per iteration (ms): 224568.4 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.16% | FLOPs estimator: workload-aware | logical tokens/s: 74708.7 | valid tokens/s: 74708.6 | physical tokens/s: 75961.8 | loss tokens/s: 60992.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.445767E-01 | lm-global-supervised-tokens: 1.369687E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.081222E-02 | lm-token-buffer-mib: 2.356907E+02 | global_load_balancing_loss: 9.997973E-01 | loss scale: 1.0 | grad norm: 0.253 | params norm: 11183.574 | params rms: 0.060555 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:05:38.903141 | finish at 2026-10-17 19:08:54 + [2026-10-14 10:06:58] iteration 1085/ 2384 | consumed samples: 34720 | elapsed time per iteration (ms): 223295.8 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 75134.5 | valid tokens/s: 75134.5 | physical tokens/s: 75934.4 | loss tokens/s: 56003.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.350496E-01 | lm-global-supervised-tokens: 1.250537E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.163528E-02 | lm-token-buffer-mib: 2.356187E+02 | global_load_balancing_loss: 1.000508E+00 | loss scale: 1.0 | grad norm: 0.303 | params norm: 11183.541 | params rms: 0.060555 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:34:21.186247 | finish at 2026-10-17 18:41:19 + [2026-10-14 10:10:40] iteration 1086/ 2384 | consumed samples: 34752 | elapsed time per iteration (ms): 222347.0 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.96% | FLOPs estimator: workload-aware | logical tokens/s: 75455.0 | valid tokens/s: 75455.0 | physical tokens/s: 75948.7 | loss tokens/s: 56357.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.826113E-01 | lm-global-supervised-tokens: 1.253088E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.499984E-02 | lm-token-buffer-mib: 2.342728E+02 | global_load_balancing_loss: 1.001277E+00 | loss scale: 1.0 | grad norm: 0.357 | params norm: 11183.504 | params rms: 0.060555 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:10:06.406158 | finish at 2026-10-17 18:20:47 + [2026-10-14 10:14:21] iteration 1087/ 2384 | consumed samples: 34784 | elapsed time per iteration (ms): 220876.8 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 75957.3 | valid tokens/s: 75957.3 | physical tokens/s: 76630.7 | loss tokens/s: 54147.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.458504E-01 | lm-global-supervised-tokens: 1.195994E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.324779E-02 | lm-token-buffer-mib: 2.340757E+02 | global_load_balancing_loss: 9.989264E-01 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11183.466 | params rms: 0.060555 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:34:37.218646 | finish at 2026-10-17 17:48:58 + [2026-10-14 10:18:02] iteration 1088/ 2384 | consumed samples: 34816 | elapsed time per iteration (ms): 220960.4 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 75928.6 | valid tokens/s: 75928.6 | physical tokens/s: 76615.1 | loss tokens/s: 57948.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.228563E-01 | lm-global-supervised-tokens: 1.280434E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.501208E-02 | lm-token-buffer-mib: 2.335543E+02 | global_load_balancing_loss: 9.996323E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11183.428 | params rms: 0.060555 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:32:44.713142 | finish at 2026-10-17 17:50:47 + [2026-10-14 10:21:48] iteration 1089/ 2384 | consumed samples: 34848 | elapsed time per iteration (ms): 226032.0 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 74224.9 | valid tokens/s: 74224.9 | physical tokens/s: 75278.2 | loss tokens/s: 59536.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.811814E-01 | lm-global-supervised-tokens: 1.345711E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.354709E-02 | lm-token-buffer-mib: 2.371779E+02 | global_load_balancing_loss: 1.002993E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11183.392 | params rms: 0.060554 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:18:31.498129 | finish at 2026-10-17 19:40:20 + [2026-10-14 10:25:57] iteration 1090/ 2384 | consumed samples: 34880 | elapsed time per iteration (ms): 248354.4 | throughput per GPU (TFLOP/s/GPU): 16.9 | MFU 5.43% | FLOPs estimator: workload-aware | logical tokens/s: 67553.4 | valid tokens/s: 67553.4 | physical tokens/s: 68538.7 | loss tokens/s: 53927.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.921599E-01 | lm-global-supervised-tokens: 1.339324E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.298130E-02 | lm-token-buffer-mib: 2.349838E+02 | global_load_balancing_loss: 1.000211E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11183.353 | params rms: 0.060554 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:16:10.651496 | finish at 2026-10-18 03:42:07 + [2026-10-14 10:29:40] iteration 1091/ 2384 | consumed samples: 34912 | elapsed time per iteration (ms): 223913.0 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 74927.4 | valid tokens/s: 74927.4 | physical tokens/s: 75573.6 | loss tokens/s: 56292.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.268439E-01 | lm-global-supervised-tokens: 1.260452E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.811391E-02 | lm-token-buffer-mib: 2.347924E+02 | global_load_balancing_loss: 1.001559E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11183.313 | params rms: 0.060554 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:25:19.493416 | finish at 2026-10-17 18:55:00 + [2026-10-14 10:33:24] iteration 1092/ 2384 | consumed samples: 34944 | elapsed time per iteration (ms): 223326.3 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 75124.0 | valid tokens/s: 75124.0 | physical tokens/s: 75958.5 | loss tokens/s: 60976.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.845236E-01 | lm-global-supervised-tokens: 1.361765E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.427085E-02 | lm-token-buffer-mib: 2.349209E+02 | global_load_balancing_loss: 1.000414E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11183.276 | params rms: 0.060554 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:08:57.518487 | finish at 2026-10-17 18:42:21 + [2026-10-14 10:37:09] iteration 1093/ 2384 | consumed samples: 34976 | elapsed time per iteration (ms): 224877.6 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 74605.9 | valid tokens/s: 74605.9 | physical tokens/s: 75214.5 | loss tokens/s: 59875.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.181280E-01 | lm-global-supervised-tokens: 1.346465E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.426322E-02 | lm-token-buffer-mib: 2.344702E+02 | global_load_balancing_loss: 9.991611E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11183.241 | params rms: 0.060554 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:38:36.923675 | finish at 2026-10-17 19:15:46 + [2026-10-14 10:40:49] iteration 1094/ 2384 | consumed samples: 35008 | elapsed time per iteration (ms): 220321.5 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 76148.6 | valid tokens/s: 76148.6 | physical tokens/s: 76957.8 | loss tokens/s: 56342.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.810268E-01 | lm-global-supervised-tokens: 1.241356E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.774087E-02 | lm-token-buffer-mib: 2.349993E+02 | global_load_balancing_loss: 1.000816E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11183.205 | params rms: 0.060553 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:56:54.763377 | finish at 2026-10-17 17:37:44 + [2026-10-14 10:44:32] iteration 1095/ 2384 | consumed samples: 35040 | elapsed time per iteration (ms): 222559.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75383.0 | valid tokens/s: 75383.0 | physical tokens/s: 76196.6 | loss tokens/s: 53294.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.213227E-01 | lm-global-supervised-tokens: 1.186126E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.633684E-02 | lm-token-buffer-mib: 2.333279E+02 | global_load_balancing_loss: 1.000585E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11183.167 | params rms: 0.060553 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:41:19.163204 | finish at 2026-10-17 18:25:51 + [2026-10-14 10:48:18] iteration 1096/ 2384 | consumed samples: 35072 | elapsed time per iteration (ms): 226830.9 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 73963.5 | valid tokens/s: 73963.5 | physical tokens/s: 75818.1 | loss tokens/s: 57064.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.757113E-01 | lm-global-supervised-tokens: 1.294406E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.464059E-02 | lm-token-buffer-mib: 2.369113E+02 | global_load_balancing_loss: 1.000995E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11183.129 | params rms: 0.060553 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:09:18.173346 | finish at 2026-10-17 19:57:37 + [2026-10-14 10:52:02] iteration 1097/ 2384 | consumed samples: 35104 | elapsed time per iteration (ms): 224121.6 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 74857.5 | valid tokens/s: 74857.5 | physical tokens/s: 75609.5 | loss tokens/s: 55444.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.182475E-01 | lm-global-supervised-tokens: 1.242625E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.290655E-02 | lm-token-buffer-mib: 2.334958E+02 | global_load_balancing_loss: 9.992878E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11183.093 | params rms: 0.060553 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:07:24.556774 | finish at 2026-10-17 18:59:27 + [2026-10-14 10:55:46] iteration 1098/ 2384 | consumed samples: 35136 | elapsed time per iteration (ms): 223606.2 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75030.1 | valid tokens/s: 75030.1 | physical tokens/s: 76063.1 | loss tokens/s: 57618.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.201472E-01 | lm-global-supervised-tokens: 1.288378E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.251981E-02 | lm-token-buffer-mib: 2.344962E+02 | global_load_balancing_loss: 9.999809E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11183.059 | params rms: 0.060553 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:52:37.598622 | finish at 2026-10-17 18:48:24 + [2026-10-14 10:59:27] iteration 1099/ 2384 | consumed samples: 35168 | elapsed time per iteration (ms): 220766.6 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 75995.2 | valid tokens/s: 75995.2 | physical tokens/s: 76522.3 | loss tokens/s: 57378.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.213689E-01 | lm-global-supervised-tokens: 1.266730E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.100772E-02 | lm-token-buffer-mib: 2.334371E+02 | global_load_balancing_loss: 1.000521E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11183.022 | params rms: 0.060552 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:48:05.094343 | finish at 2026-10-17 17:47:32 + [2026-10-14 11:03:31] iteration 1100/ 2384 | consumed samples: 35200 | elapsed time per iteration (ms): 244373.3 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 68653.9 | valid tokens/s: 68653.9 | physical tokens/s: 69313.8 | loss tokens/s: 53044.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.579645E-01 | lm-global-supervised-tokens: 1.296263E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.196061E-02 | lm-token-buffer-mib: 2.339234E+02 | global_load_balancing_loss: 1.000565E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11182.988 | params rms: 0.060552 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:09:35.293113 | finish at 2026-10-18 02:13:07 + [2026-10-14 11:07:15] iteration 1101/ 2384 | consumed samples: 35232 | elapsed time per iteration (ms): 223406.2 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 75097.2 | valid tokens/s: 75097.2 | physical tokens/s: 76199.1 | loss tokens/s: 59573.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.904733E-01 | lm-global-supervised-tokens: 1.330917E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.383543E-02 | lm-token-buffer-mib: 2.359150E+02 | global_load_balancing_loss: 9.991226E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11182.954 | params rms: 0.060552 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:37:10.148395 | finish at 2026-10-17 18:44:25 + [2026-10-14 11:10:56] iteration 1102/ 2384 | consumed samples: 35264 | elapsed time per iteration (ms): 221838.9 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75627.9 | valid tokens/s: 75627.9 | physical tokens/s: 76257.1 | loss tokens/s: 55831.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.429687E-01 | lm-global-supervised-tokens: 1.238559E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.929663E-02 | lm-token-buffer-mib: 2.333137E+02 | global_load_balancing_loss: 1.001863E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11182.921 | params rms: 0.060552 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:59:57.483363 | finish at 2026-10-17 18:10:54 + [2026-10-14 11:14:37] iteration 1103/ 2384 | consumed samples: 35296 | elapsed time per iteration (ms): 220858.9 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 75963.3 | valid tokens/s: 75963.3 | physical tokens/s: 76490.9 | loss tokens/s: 55666.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.674177E-01 | lm-global-supervised-tokens: 1.229451E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.170615E-02 | lm-token-buffer-mib: 2.346090E+02 | global_load_balancing_loss: 1.001233E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11182.885 | params rms: 0.060552 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:35:20.260763 | finish at 2026-10-17 17:49:58 + [2026-10-14 11:18:21] iteration 1104/ 2384 | consumed samples: 35328 | elapsed time per iteration (ms): 223366.4 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 75110.4 | valid tokens/s: 75110.4 | physical tokens/s: 75662.0 | loss tokens/s: 54272.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.353112E-01 | lm-global-supervised-tokens: 1.212268E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.771377E-02 | lm-token-buffer-mib: 2.344925E+02 | global_load_balancing_loss: 1.001310E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11182.845 | params rms: 0.060551 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:25:08.955383 | finish at 2026-10-17 18:43:30 + [2026-10-14 11:22:03] iteration 1105/ 2384 | consumed samples: 35360 | elapsed time per iteration (ms): 222317.9 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 75464.9 | valid tokens/s: 75464.9 | physical tokens/s: 75895.9 | loss tokens/s: 59864.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.259885E-01 | lm-global-supervised-tokens: 1.330884E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.263834E-02 | lm-token-buffer-mib: 2.328177E+02 | global_load_balancing_loss: 9.999495E-01 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11182.804 | params rms: 0.060551 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:59:04.648000 | finish at 2026-10-17 18:21:08 + [2026-10-14 11:25:49] iteration 1106/ 2384 | consumed samples: 35392 | elapsed time per iteration (ms): 226455.0 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 74086.1 | valid tokens/s: 74086.1 | physical tokens/s: 74913.3 | loss tokens/s: 53823.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.162552E-01 | lm-global-supervised-tokens: 1.218851E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.068851E-02 | lm-token-buffer-mib: 2.358022E+02 | global_load_balancing_loss: 1.000914E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11182.763 | params rms: 0.060551 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:23:29.440541 | finish at 2026-10-17 19:49:19 + [2026-10-14 11:29:32] iteration 1107/ 2384 | consumed samples: 35424 | elapsed time per iteration (ms): 222624.9 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 75360.8 | valid tokens/s: 75360.8 | physical tokens/s: 76708.6 | loss tokens/s: 59413.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.855042E-01 | lm-global-supervised-tokens: 1.322683E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.354562E-02 | lm-token-buffer-mib: 2.336096E+02 | global_load_balancing_loss: 1.000378E+00 | loss scale: 1.0 | grad norm: 0.406 | params norm: 11182.726 | params rms: 0.060551 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:58:12.026355 | finish at 2026-10-17 18:27:44 + [2026-10-14 11:33:13] iteration 1108/ 2384 | consumed samples: 35456 | elapsed time per iteration (ms): 220735.5 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 76005.9 | valid tokens/s: 76005.9 | physical tokens/s: 76706.2 | loss tokens/s: 56090.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.455269E-01 | lm-global-supervised-tokens: 1.238108E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.308458E-02 | lm-token-buffer-mib: 2.322646E+02 | global_load_balancing_loss: 1.000432E+00 | loss scale: 1.0 | grad norm: 0.340 | params norm: 11182.690 | params rms: 0.060551 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:14:18.537369 | finish at 2026-10-17 17:47:31 + [2026-10-14 11:36:57] iteration 1109/ 2384 | consumed samples: 35488 | elapsed time per iteration (ms): 224333.7 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 74786.7 | valid tokens/s: 74786.7 | physical tokens/s: 75524.9 | loss tokens/s: 54963.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.096376E-01 | lm-global-supervised-tokens: 1.233018E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.617658E-02 | lm-token-buffer-mib: 2.350601E+02 | global_load_balancing_loss: 9.994410E-01 | loss scale: 1.0 | grad norm: 0.365 | params norm: 11182.653 | params rms: 0.060550 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:27:05.413316 | finish at 2026-10-17 19:04:03 + [2026-10-14 11:41:03] iteration 1110/ 2384 | consumed samples: 35520 | elapsed time per iteration (ms): 246216.2 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 68140.0 | valid tokens/s: 68140.0 | physical tokens/s: 69113.6 | loss tokens/s: 54061.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.505387E-01 | lm-global-supervised-tokens: 1.331089E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.693429E-02 | lm-token-buffer-mib: 2.363725E+02 | global_load_balancing_loss: 9.998298E-01 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11182.614 | params rms: 0.060550 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:07:59.436514 | finish at 2026-10-18 02:49:03 + [2026-10-14 11:44:48] iteration 1111/ 2384 | consumed samples: 35552 | elapsed time per iteration (ms): 224617.1 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 74692.4 | valid tokens/s: 74692.3 | physical tokens/s: 75611.4 | loss tokens/s: 56868.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.769803E-01 | lm-global-supervised-tokens: 1.277366E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.393371E-02 | lm-token-buffer-mib: 2.343133E+02 | global_load_balancing_loss: 9.997798E-01 | loss scale: 1.0 | grad norm: 0.580 | params norm: 11182.573 | params rms: 0.060550 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:25:37.545537 | finish at 2026-10-17 19:10:26 + [2026-10-14 11:48:32] iteration 1112/ 2384 | consumed samples: 35584 | elapsed time per iteration (ms): 223836.1 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 74952.9 | valid tokens/s: 74952.9 | physical tokens/s: 75882.1 | loss tokens/s: 55709.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.090871E-01 | lm-global-supervised-tokens: 1.246976E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.926113E-02 | lm-token-buffer-mib: 2.352331E+02 | global_load_balancing_loss: 1.000101E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11182.532 | params rms: 0.060550 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:05:19.549353 | finish at 2026-10-17 18:53:51 + [2026-10-14 11:52:14] iteration 1113/ 2384 | consumed samples: 35616 | elapsed time per iteration (ms): 222591.5 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 75372.0 | valid tokens/s: 75372.0 | physical tokens/s: 76391.5 | loss tokens/s: 58969.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.175312E-01 | lm-global-supervised-tokens: 1.312618E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.372049E-02 | lm-token-buffer-mib: 2.351252E+02 | global_load_balancing_loss: 1.003291E+00 | loss scale: 1.0 | grad norm: 0.662 | params norm: 11182.492 | params rms: 0.060550 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:35:13.808374 | finish at 2026-10-17 18:27:28 + [2026-10-14 11:55:57] iteration 1114/ 2384 | consumed samples: 35648 | elapsed time per iteration (ms): 222932.5 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 75256.8 | valid tokens/s: 75256.8 | physical tokens/s: 76068.2 | loss tokens/s: 56696.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.400720E-01 | lm-global-supervised-tokens: 1.263959E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.341524E-02 | lm-token-buffer-mib: 2.342986E+02 | global_load_balancing_loss: 1.001170E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11182.451 | params rms: 0.060549 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:38:44.265771 | finish at 2026-10-17 18:34:42 + [2026-10-14 11:59:40] iteration 1115/ 2384 | consumed samples: 35680 | elapsed time per iteration (ms): 222211.1 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75501.1 | valid tokens/s: 75501.1 | physical tokens/s: 76459.8 | loss tokens/s: 57996.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.601749E-01 | lm-global-supervised-tokens: 1.288752E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.035921E-02 | lm-token-buffer-mib: 2.341079E+02 | global_load_balancing_loss: 1.000055E+00 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11182.406 | params rms: 0.060549 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:19:45.915982 | finish at 2026-10-17 18:19:25 + [2026-10-14 12:03:21] iteration 1116/ 2384 | consumed samples: 35712 | elapsed time per iteration (ms): 220959.2 | throughput per GPU (TFLOP/s/GPU): 18.1 | MFU 5.81% | FLOPs estimator: workload-aware | logical tokens/s: 75929.0 | valid tokens/s: 75929.0 | physical tokens/s: 77208.7 | loss tokens/s: 57514.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.584048E-01 | lm-global-supervised-tokens: 1.270826E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.336531E-02 | lm-token-buffer-mib: 2.357101E+02 | global_load_balancing_loss: 9.996840E-01 | loss scale: 1.0 | grad norm: 0.474 | params norm: 11182.362 | params rms: 0.060549 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:49:36.229807 | finish at 2026-10-17 17:52:57 + [2026-10-14 12:06:59] iteration 1117/ 2384 | consumed samples: 35744 | elapsed time per iteration (ms): 218916.6 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.01% | FLOPs estimator: workload-aware | logical tokens/s: 76637.3 | valid tokens/s: 76637.3 | physical tokens/s: 76760.3 | loss tokens/s: 53625.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.463943E-01 | lm-global-supervised-tokens: 1.173957E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.135894E-02 | lm-token-buffer-mib: 2.325615E+02 | global_load_balancing_loss: 9.998340E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11182.320 | params rms: 0.060549 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:02:47.369341 | finish at 2026-10-17 17:09:47 + [2026-10-14 12:10:42] iteration 1118/ 2384 | consumed samples: 35776 | elapsed time per iteration (ms): 223027.8 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.88% | FLOPs estimator: workload-aware | logical tokens/s: 75224.6 | valid tokens/s: 75224.6 | physical tokens/s: 75864.4 | loss tokens/s: 57624.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.677345E-01 | lm-global-supervised-tokens: 1.285190E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.441594E-02 | lm-token-buffer-mib: 2.337331E+02 | global_load_balancing_loss: 1.001207E+00 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11182.282 | params rms: 0.060548 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:25:53.248632 | finish at 2026-10-17 18:36:36 + [2026-10-14 12:14:28] iteration 1119/ 2384 | consumed samples: 35808 | elapsed time per iteration (ms): 225513.4 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 74395.6 | valid tokens/s: 74395.6 | physical tokens/s: 75063.4 | loss tokens/s: 54845.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.694006E-01 | lm-global-supervised-tokens: 1.236837E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.680098E-02 | lm-token-buffer-mib: 2.351963E+02 | global_load_balancing_loss: 9.990726E-01 | loss scale: 1.0 | grad norm: 0.194 | params norm: 11182.242 | params rms: 0.060548 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:14:34.443257 | finish at 2026-10-17 19:29:02 + [2026-10-14 12:18:34] iteration 1120/ 2384 | consumed samples: 35840 | elapsed time per iteration (ms): 245955.4 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 68212.3 | valid tokens/s: 68212.3 | physical tokens/s: 68744.8 | loss tokens/s: 52032.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.190178E-01 | lm-global-supervised-tokens: 1.279776E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.627061E-02 | lm-token-buffer-mib: 2.335958E+02 | global_load_balancing_loss: 1.000460E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11182.198 | params rms: 0.060548 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:21:27.564110 | finish at 2026-10-18 02:40:02 +times across ranks (ms): + save-checkpoint: + rank 0: 24844.15 + rank 1: 24844.13 + rank 2: 24844.17 + rank 3: 24844.16 + rank 4: 24844.12 + rank 5: 24844.13 + rank 6: 24844.15 + rank 7: 24844.14 + rank 8: 24844.18 + rank 9: 24844.21 + rank 10: 24844.21 + rank 11: 24844.18 + rank 12: 24844.19 + rank 13: 24844.18 + rank 14: 24844.18 + rank 15: 24844.19 + rank 16: 24843.42 + rank 17: 24843.41 + rank 18: 24843.43 + rank 19: 24843.41 + rank 20: 24843.40 + rank 21: 24843.40 + rank 22: 24843.27 + rank 23: 24843.41 + rank 24: 24844.29 + rank 25: 24844.27 + rank 26: 24844.35 + rank 27: 24845.02 + rank 28: 24844.26 + rank 29: 24844.29 + rank 30: 24844.31 + rank 31: 24844.28 + rank 32: 24844.34 + rank 33: 24844.35 + rank 34: 24844.33 + rank 35: 24844.29 + rank 36: 24844.39 + rank 37: 24844.24 + rank 38: 24844.34 + rank 39: 24844.36 + rank 40: 24844.04 + rank 41: 24844.04 + rank 42: 24844.11 + rank 43: 24844.05 + rank 44: 24844.03 + rank 45: 24844.02 + rank 46: 24844.03 + rank 47: 24844.04 + rank 48: 24843.89 + rank 49: 24843.79 + rank 50: 24843.76 + rank 51: 24843.81 + rank 52: 24843.82 + rank 53: 24843.89 + rank 54: 24843.76 + rank 55: 24843.82 + rank 56: 24844.00 + rank 57: 24844.03 + rank 58: 24844.03 + rank 59: 24844.04 + rank 60: 24844.04 + rank 61: 24843.99 + rank 62: 24843.99 + rank 63: 24843.98 + [2026-10-14 12:22:46] iteration 1121/ 2384 | consumed samples: 35872 | elapsed time per iteration (ms): 227250.7 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 73826.8 | valid tokens/s: 73826.8 | physical tokens/s: 74351.6 | loss tokens/s: 56061.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.793510E-01 | lm-global-supervised-tokens: 1.273995E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.616891E-02 | lm-token-buffer-mib: 2.341148E+02 | global_load_balancing_loss: 9.998733E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11182.152 | params rms: 0.060548 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:43:37.638613 | finish at 2026-10-17 20:06:24 + failed to log checkpoint /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0001120 as wandb artifact: Unable to connect to server to verify API token. + [2026-10-14 12:26:38] iteration 1122/ 2384 | consumed samples: 35904 | elapsed time per iteration (ms): 232294.0 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 72224.0 | valid tokens/s: 72224.0 | physical tokens/s: 72781.1 | loss tokens/s: 51704.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.128825E-01 | lm-global-supervised-tokens: 1.201058E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.072512E-02 | lm-token-buffer-mib: 2.327504E+02 | global_load_balancing_loss: 1.001018E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11182.111 | params rms: 0.060547 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:25:55.046241 | finish at 2026-10-17 21:52:33 + [2026-10-14 12:30:21] iteration 1123/ 2384 | consumed samples: 35936 | elapsed time per iteration (ms): 222159.7 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 75518.6 | valid tokens/s: 75518.6 | physical tokens/s: 76124.9 | loss tokens/s: 53748.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.813814E-01 | lm-global-supervised-tokens: 1.194077E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.601266E-02 | lm-token-buffer-mib: 2.323758E+02 | global_load_balancing_loss: 1.001514E+00 | loss scale: 1.0 | grad norm: 0.239 | params norm: 11182.072 | params rms: 0.060547 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:49:03.328982 | finish at 2026-10-17 18:19:24 + [2026-10-14 12:34:00] iteration 1124/ 2384 | consumed samples: 35968 | elapsed time per iteration (ms): 219044.1 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 76592.8 | valid tokens/s: 76592.8 | physical tokens/s: 76851.2 | loss tokens/s: 55658.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.692767E-01 | lm-global-supervised-tokens: 1.219168E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.513077E-02 | lm-token-buffer-mib: 2.328722E+02 | global_load_balancing_loss: 9.998939E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11182.031 | params rms: 0.060547 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:39:55.509882 | finish at 2026-10-17 17:13:55 + [2026-10-14 12:37:42] iteration 1125/ 2384 | consumed samples: 36000 | elapsed time per iteration (ms): 222690.0 | throughput per GPU (TFLOP/s/GPU): 27.8 | MFU 8.90% | FLOPs estimator: workload-aware | logical tokens/s: 75338.8 | valid tokens/s: 75338.8 | physical tokens/s: 75699.6 | loss tokens/s: 54627.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.337592E-01 | lm-global-supervised-tokens: 1.216495E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.268941E-02 | lm-token-buffer-mib: 2.338721E+02 | global_load_balancing_loss: 1.000721E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11181.987 | params rms: 0.060547 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:52:46.650038 | finish at 2026-10-17 18:30:29 + [2026-10-14 12:41:22] iteration 1126/ 2384 | consumed samples: 36032 | elapsed time per iteration (ms): 219827.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 76320.0 | valid tokens/s: 76320.0 | physical tokens/s: 77024.1 | loss tokens/s: 58701.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.230751E-01 | lm-global-supervised-tokens: 1.290411E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.217936E-02 | lm-token-buffer-mib: 2.327385E+02 | global_load_balancing_loss: 1.000338E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11181.945 | params rms: 0.060547 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:49:02.349681 | finish at 2026-10-17 17:30:24 + [2026-10-14 12:45:04] iteration 1127/ 2384 | consumed samples: 36064 | elapsed time per iteration (ms): 221615.7 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 75703.9 | valid tokens/s: 75703.9 | physical tokens/s: 76482.7 | loss tokens/s: 58279.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.379575E-01 | lm-global-supervised-tokens: 1.291564E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.233308E-02 | lm-token-buffer-mib: 2.338021E+02 | global_load_balancing_loss: 9.996375E-01 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11181.904 | params rms: 0.060546 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:22:50.963379 | finish at 2026-10-17 18:07:55 + [2026-10-14 12:48:48] iteration 1128/ 2384 | consumed samples: 36096 | elapsed time per iteration (ms): 223992.8 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 74900.6 | valid tokens/s: 74900.6 | physical tokens/s: 76239.0 | loss tokens/s: 59866.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.220625E-01 | lm-global-supervised-tokens: 1.340966E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.927867E-02 | lm-token-buffer-mib: 2.360912E+02 | global_load_balancing_loss: 1.000002E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11181.862 | params rms: 0.060546 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:08:54.952904 | finish at 2026-10-17 18:57:43 + [2026-10-14 12:52:33] iteration 1129/ 2384 | consumed samples: 36128 | elapsed time per iteration (ms): 225786.9 | throughput per GPU (TFLOP/s/GPU): 26.4 | MFU 8.46% | FLOPs estimator: workload-aware | logical tokens/s: 74305.4 | valid tokens/s: 74305.4 | physical tokens/s: 74691.0 | loss tokens/s: 55320.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.181727E-01 | lm-global-supervised-tokens: 1.249054E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.178050E-02 | lm-token-buffer-mib: 2.337037E+02 | global_load_balancing_loss: 1.000409E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11181.822 | params rms: 0.060546 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:42:42.511381 | finish at 2026-10-17 19:35:16 + [2026-10-14 12:56:35] iteration 1130/ 2384 | consumed samples: 36160 | elapsed time per iteration (ms): 241118.9 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 69580.5 | valid tokens/s: 69580.5 | physical tokens/s: 70010.4 | loss tokens/s: 52707.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.024208E-01 | lm-global-supervised-tokens: 1.270877E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.213890E-02 | lm-token-buffer-mib: 2.332341E+02 | global_load_balancing_loss: 1.000672E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11181.780 | params rms: 0.060546 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:59:23.112336 | finish at 2026-10-18 00:55:58 + [2026-10-14 13:00:16] iteration 1131/ 2384 | consumed samples: 36192 | elapsed time per iteration (ms): 221599.7 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75709.5 | valid tokens/s: 75709.5 | physical tokens/s: 76335.1 | loss tokens/s: 58007.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.081181E-01 | lm-global-supervised-tokens: 1.285448E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.213198E-02 | lm-token-buffer-mib: 2.328573E+02 | global_load_balancing_loss: 1.000414E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11181.737 | params rms: 0.060545 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:07:44.486504 | finish at 2026-10-17 18:08:01 + [2026-10-14 13:03:59] iteration 1132/ 2384 | consumed samples: 36224 | elapsed time per iteration (ms): 223055.8 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.25% | FLOPs estimator: workload-aware | logical tokens/s: 75215.2 | valid tokens/s: 75215.2 | physical tokens/s: 75934.9 | loss tokens/s: 58496.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.480582E-01 | lm-global-supervised-tokens: 1.304790E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.636865E-02 | lm-token-buffer-mib: 2.340242E+02 | global_load_balancing_loss: 9.999846E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11181.696 | params rms: 0.060545 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:34:25.866029 | finish at 2026-10-17 18:38:25 + [2026-10-14 13:07:41] iteration 1133/ 2384 | consumed samples: 36256 | elapsed time per iteration (ms): 222097.6 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 75539.7 | valid tokens/s: 75539.7 | physical tokens/s: 75962.6 | loss tokens/s: 54642.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.312254E-01 | lm-global-supervised-tokens: 1.213587E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.312799E-02 | lm-token-buffer-mib: 2.326147E+02 | global_load_balancing_loss: 1.000707E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11181.656 | params rms: 0.060545 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:10:44.082666 | finish at 2026-10-17 18:18:25 + [2026-10-14 13:11:23] iteration 1134/ 2384 | consumed samples: 36288 | elapsed time per iteration (ms): 221530.8 | throughput per GPU (TFLOP/s/GPU): 26.0 | MFU 8.33% | FLOPs estimator: workload-aware | logical tokens/s: 75733.0 | valid tokens/s: 75733.0 | physical tokens/s: 76028.7 | loss tokens/s: 51773.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.950815E-01 | lm-global-supervised-tokens: 1.146952E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.752122E-02 | lm-token-buffer-mib: 2.325392E+02 | global_load_balancing_loss: 1.000108E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11181.616 | params rms: 0.060545 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:55:13.439631 | finish at 2026-10-17 18:06:36 + [2026-10-14 13:15:11] iteration 1135/ 2384 | consumed samples: 36320 | elapsed time per iteration (ms): 228600.6 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 73390.9 | valid tokens/s: 73390.9 | physical tokens/s: 74699.2 | loss tokens/s: 56377.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.071818E-01 | lm-global-supervised-tokens: 1.288795E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.337990E-02 | lm-token-buffer-mib: 2.381560E+02 | global_load_balancing_loss: 1.001302E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11181.575 | params rms: 0.060545 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:18:42.087466 | finish at 2026-10-17 20:33:54 + [2026-10-14 13:18:57] iteration 1136/ 2384 | consumed samples: 36352 | elapsed time per iteration (ms): 225445.6 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.53% | FLOPs estimator: workload-aware | logical tokens/s: 74418.0 | valid tokens/s: 74418.0 | physical tokens/s: 75298.9 | loss tokens/s: 54116.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.754291E-01 | lm-global-supervised-tokens: 1.220020E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.574330E-02 | lm-token-buffer-mib: 2.362558E+02 | global_load_balancing_loss: 9.999603E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11181.537 | params rms: 0.060544 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:09:16.063316 | finish at 2026-10-17 19:28:13 + [2026-10-14 13:22:40] iteration 1137/ 2384 | consumed samples: 36384 | elapsed time per iteration (ms): 223173.6 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 75175.6 | valid tokens/s: 75175.6 | physical tokens/s: 75596.0 | loss tokens/s: 56049.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.563683E-01 | lm-global-supervised-tokens: 1.250872E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.947340E-02 | lm-token-buffer-mib: 2.336648E+02 | global_load_balancing_loss: 1.001735E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11181.502 | params rms: 0.060544 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:18:17.526420 | finish at 2026-10-17 18:40:58 + [2026-10-14 13:26:20] iteration 1138/ 2384 | consumed samples: 36416 | elapsed time per iteration (ms): 220043.8 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 76244.8 | valid tokens/s: 76244.8 | physical tokens/s: 76858.0 | loss tokens/s: 57938.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.169900E-01 | lm-global-supervised-tokens: 1.274892E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.348952E-02 | lm-token-buffer-mib: 2.337864E+02 | global_load_balancing_loss: 1.001195E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11181.468 | params rms: 0.060544 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:09:34.522363 | finish at 2026-10-17 17:35:55 + [2026-10-14 13:30:01] iteration 1139/ 2384 | consumed samples: 36448 | elapsed time per iteration (ms): 221045.0 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 75899.5 | valid tokens/s: 75899.5 | physical tokens/s: 76528.4 | loss tokens/s: 57215.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.666632E-01 | lm-global-supervised-tokens: 1.264730E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.232658E-02 | lm-token-buffer-mib: 2.340295E+02 | global_load_balancing_loss: 1.000953E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11181.433 | params rms: 0.060544 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:26:41.024204 | finish at 2026-10-17 17:56:42 + [2026-10-14 13:34:03] iteration 1140/ 2384 | consumed samples: 36480 | elapsed time per iteration (ms): 242299.1 | throughput per GPU (TFLOP/s/GPU): 17.8 | MFU 5.70% | FLOPs estimator: workload-aware | logical tokens/s: 69241.8 | valid tokens/s: 69241.7 | physical tokens/s: 70348.5 | loss tokens/s: 55403.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.606672E-01 | lm-global-supervised-tokens: 1.342419E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.914183E-02 | lm-token-buffer-mib: 2.359097E+02 | global_load_balancing_loss: 9.994892E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11181.395 | params rms: 0.060544 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:43:40.102251 | finish at 2026-10-18 01:17:44 + [2026-10-14 13:37:44] iteration 1141/ 2384 | consumed samples: 36512 | elapsed time per iteration (ms): 220925.5 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75940.5 | valid tokens/s: 75940.5 | physical tokens/s: 76814.6 | loss tokens/s: 57240.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.006564E-01 | lm-global-supervised-tokens: 1.264585E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.358636E-02 | lm-token-buffer-mib: 2.338711E+02 | global_load_balancing_loss: 1.000044E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11181.365 | params rms: 0.060543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:16:50.458630 | finish at 2026-10-17 17:54:35 + [2026-10-14 13:41:33] iteration 1142/ 2384 | consumed samples: 36544 | elapsed time per iteration (ms): 228916.4 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 73289.5 | valid tokens/s: 73289.5 | physical tokens/s: 74581.9 | loss tokens/s: 55213.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.172254E-01 | lm-global-supervised-tokens: 1.263918E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.297668E-02 | lm-token-buffer-mib: 2.371159E+02 | global_load_balancing_loss: 9.989482E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11181.333 | params rms: 0.060543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:58:34.199541 | finish at 2026-10-17 20:40:08 + [2026-10-14 13:45:20] iteration 1143/ 2384 | consumed samples: 36576 | elapsed time per iteration (ms): 226191.6 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.23% | FLOPs estimator: workload-aware | logical tokens/s: 74172.5 | valid tokens/s: 74172.5 | physical tokens/s: 75155.5 | loss tokens/s: 59791.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.935408E-01 | lm-global-supervised-tokens: 1.352424E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.633905E-02 | lm-token-buffer-mib: 2.346249E+02 | global_load_balancing_loss: 9.996061E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11181.300 | params rms: 0.060543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:58:23.758248 | finish at 2026-10-17 19:43:43 + [2026-10-14 13:49:06] iteration 1144/ 2384 | consumed samples: 36608 | elapsed time per iteration (ms): 226470.9 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 74081.0 | valid tokens/s: 74081.0 | physical tokens/s: 75541.6 | loss tokens/s: 56145.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.321909E-01 | lm-global-supervised-tokens: 1.271520E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.262917E-02 | lm-token-buffer-mib: 2.357102E+02 | global_load_balancing_loss: 1.000461E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11181.264 | params rms: 0.060543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:00:23.858128 | finish at 2026-10-17 19:49:30 + [2026-10-14 13:52:51] iteration 1145/ 2384 | consumed samples: 36640 | elapsed time per iteration (ms): 224867.8 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 74609.2 | valid tokens/s: 74609.2 | physical tokens/s: 76187.7 | loss tokens/s: 55309.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.116278E-01 | lm-global-supervised-tokens: 1.243741E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.102012E-02 | lm-token-buffer-mib: 2.355402E+02 | global_load_balancing_loss: 1.001287E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11181.229 | params rms: 0.060543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:23:31.192971 | finish at 2026-10-17 19:16:22 + [2026-10-14 13:56:34] iteration 1146/ 2384 | consumed samples: 36672 | elapsed time per iteration (ms): 223238.8 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 75153.7 | valid tokens/s: 75153.7 | physical tokens/s: 75850.9 | loss tokens/s: 58966.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.787140E-01 | lm-global-supervised-tokens: 1.316355E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.344360E-02 | lm-token-buffer-mib: 2.346933E+02 | global_load_balancing_loss: 1.002292E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11181.195 | params rms: 0.060543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:46:09.684638 | finish at 2026-10-17 18:42:44 + [2026-10-14 14:00:14] iteration 1147/ 2384 | consumed samples: 36704 | elapsed time per iteration (ms): 220331.9 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.13% | FLOPs estimator: workload-aware | logical tokens/s: 76145.1 | valid tokens/s: 76145.1 | physical tokens/s: 76721.8 | loss tokens/s: 55970.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.483879E-01 | lm-global-supervised-tokens: 1.233198E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.402253E-02 | lm-token-buffer-mib: 2.333714E+02 | global_load_balancing_loss: 1.002085E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11181.161 | params rms: 0.060542 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:42:30.547472 | finish at 2026-10-17 17:42:45 + [2026-10-14 14:04:02] iteration 1148/ 2384 | consumed samples: 36736 | elapsed time per iteration (ms): 227313.0 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.90% | FLOPs estimator: workload-aware | logical tokens/s: 73806.5 | valid tokens/s: 73806.5 | physical tokens/s: 75375.5 | loss tokens/s: 56492.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.299155E-01 | lm-global-supervised-tokens: 1.284136E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.429439E-02 | lm-token-buffer-mib: 2.363279E+02 | global_load_balancing_loss: 1.000585E+00 | loss scale: 1.0 | grad norm: 0.258 | params norm: 11181.124 | params rms: 0.060542 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:02:38.851158 | finish at 2026-10-17 20:06:41 + [2026-10-14 14:07:43] iteration 1149/ 2384 | consumed samples: 36768 | elapsed time per iteration (ms): 221308.8 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.24% | FLOPs estimator: workload-aware | logical tokens/s: 75809.0 | valid tokens/s: 75809.0 | physical tokens/s: 76714.5 | loss tokens/s: 56665.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.711078E-01 | lm-global-supervised-tokens: 1.254067E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.364505E-02 | lm-token-buffer-mib: 2.336713E+02 | global_load_balancing_loss: 9.992006E-01 | loss scale: 1.0 | grad norm: 0.369 | params norm: 11181.087 | params rms: 0.060542 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:55:16.388589 | finish at 2026-10-17 18:02:59 + [2026-10-14 14:11:48] iteration 1150/ 2384 | consumed samples: 36800 | elapsed time per iteration (ms): 244967.7 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 68487.4 | valid tokens/s: 68487.4 | physical tokens/s: 69324.0 | loss tokens/s: 50763.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.103194E-01 | lm-global-supervised-tokens: 1.243533E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.618918E-02 | lm-token-buffer-mib: 2.344558E+02 | global_load_balancing_loss: 9.992638E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11181.049 | params rms: 0.060542 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:58:10.172403 | finish at 2026-10-18 02:09:58 + [2026-10-14 14:15:34] iteration 1151/ 2384 | consumed samples: 36832 | elapsed time per iteration (ms): 225762.7 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 74313.5 | valid tokens/s: 74313.5 | physical tokens/s: 75228.0 | loss tokens/s: 56756.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.184976E-01 | lm-global-supervised-tokens: 1.281353E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.562224E-02 | lm-token-buffer-mib: 2.354133E+02 | global_load_balancing_loss: 9.991621E-01 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11181.012 | params rms: 0.060542 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:19:25.427132 | finish at 2026-10-17 19:34:59 + [2026-10-14 14:19:17] iteration 1152/ 2384 | consumed samples: 36864 | elapsed time per iteration (ms): 223352.7 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.48% | FLOPs estimator: workload-aware | logical tokens/s: 75115.2 | valid tokens/s: 75115.2 | physical tokens/s: 75822.8 | loss tokens/s: 54785.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.233636E-01 | lm-global-supervised-tokens: 1.223649E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.779257E-02 | lm-token-buffer-mib: 2.350063E+02 | global_load_balancing_loss: 9.999164E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11180.978 | params rms: 0.060541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:26:10.569572 | finish at 2026-10-17 18:45:28 + [2026-10-14 14:22:58] iteration 1153/ 2384 | consumed samples: 36896 | elapsed time per iteration (ms): 220442.8 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 76106.7 | valid tokens/s: 76106.7 | physical tokens/s: 76713.1 | loss tokens/s: 57688.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.264985E-01 | lm-global-supervised-tokens: 1.271710E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.337887E-02 | lm-token-buffer-mib: 2.347384E+02 | global_load_balancing_loss: 1.000317E+00 | loss scale: 1.0 | grad norm: 0.262 | params norm: 11180.947 | params rms: 0.060541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:22:45.028450 | finish at 2026-10-17 17:45:43 + [2026-10-14 14:26:42] iteration 1154/ 2384 | consumed samples: 36928 | elapsed time per iteration (ms): 224625.0 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 74689.8 | valid tokens/s: 74689.8 | physical tokens/s: 75491.9 | loss tokens/s: 54279.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.363433E-01 | lm-global-supervised-tokens: 1.219253E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.898862E-02 | lm-token-buffer-mib: 2.348145E+02 | global_load_balancing_loss: 1.000597E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11180.917 | params rms: 0.060541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:44:48.772581 | finish at 2026-10-17 19:11:31 + [2026-10-14 14:30:26] iteration 1155/ 2384 | consumed samples: 36960 | elapsed time per iteration (ms): 223932.3 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.48% | FLOPs estimator: workload-aware | logical tokens/s: 74920.6 | valid tokens/s: 74920.6 | physical tokens/s: 75584.6 | loss tokens/s: 50735.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.473213E-01 | lm-global-supervised-tokens: 1.136131E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.374718E-02 | lm-token-buffer-mib: 2.333868E+02 | global_load_balancing_loss: 1.002134E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11180.888 | params rms: 0.060541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:26:52.750515 | finish at 2026-10-17 18:57:19 + [2026-10-14 14:34:08] iteration 1156/ 2384 | consumed samples: 36992 | elapsed time per iteration (ms): 221673.4 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 75684.3 | valid tokens/s: 75684.3 | physical tokens/s: 77055.8 | loss tokens/s: 61182.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.588520E-01 | lm-global-supervised-tokens: 1.356245E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.896205E-02 | lm-token-buffer-mib: 2.329337E+02 | global_load_balancing_loss: 1.001740E+00 | loss scale: 1.0 | grad norm: 0.308 | params norm: 11180.856 | params rms: 0.060541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:36:54.949747 | finish at 2026-10-17 18:11:03 + [2026-10-14 14:37:51] iteration 1157/ 2384 | consumed samples: 37024 | elapsed time per iteration (ms): 223050.7 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 75216.8 | valid tokens/s: 75216.8 | physical tokens/s: 75594.4 | loss tokens/s: 58021.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.339541E-01 | lm-global-supervised-tokens: 1.294167E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.705617E-02 | lm-token-buffer-mib: 2.335808E+02 | global_load_balancing_loss: 1.000145E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11180.824 | params rms: 0.060541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:01:23.158228 | finish at 2026-10-17 18:39:14 + [2026-10-14 14:41:36] iteration 1158/ 2384 | consumed samples: 37056 | elapsed time per iteration (ms): 225301.6 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.75% | FLOPs estimator: workload-aware | logical tokens/s: 74465.4 | valid tokens/s: 74465.4 | physical tokens/s: 75316.7 | loss tokens/s: 56142.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.030265E-01 | lm-global-supervised-tokens: 1.264891E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.495431E-02 | lm-token-buffer-mib: 2.347270E+02 | global_load_balancing_loss: 1.000964E+00 | loss scale: 1.0 | grad norm: 0.752 | params norm: 11180.788 | params rms: 0.060540 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:43:39.718314 | finish at 2026-10-17 19:25:16 + [2026-10-14 14:45:20] iteration 1159/ 2384 | consumed samples: 37088 | elapsed time per iteration (ms): 223823.5 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 74957.3 | valid tokens/s: 74957.3 | physical tokens/s: 75855.8 | loss tokens/s: 57210.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.074791E-01 | lm-global-supervised-tokens: 1.280504E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.845989E-02 | lm-token-buffer-mib: 2.336324E+02 | global_load_balancing_loss: 9.999591E-01 | loss scale: 1.0 | grad norm: 0.219 | params norm: 11180.751 | params rms: 0.060540 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:09:43.747095 | finish at 2026-10-17 18:55:04 + [2026-10-14 14:49:28] iteration 1160/ 2384 | consumed samples: 37120 | elapsed time per iteration (ms): 247663.0 | throughput per GPU (TFLOP/s/GPU): 16.3 | MFU 5.22% | FLOPs estimator: workload-aware | logical tokens/s: 67742.1 | valid tokens/s: 67742.1 | physical tokens/s: 68982.5 | loss tokens/s: 55098.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.617164E-01 | lm-global-supervised-tokens: 1.364581E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.784115E-02 | lm-token-buffer-mib: 2.393279E+02 | global_load_balancing_loss: 9.997038E-01 | loss scale: 1.0 | grad norm: 0.615 | params norm: 11180.716 | params rms: 0.060540 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:12:19.456392 | finish at 2026-10-18 03:01:47 +times across ranks (ms): + save-checkpoint: + rank 0: 24825.88 + rank 1: 24825.90 + rank 2: 24825.90 + rank 3: 24825.89 + rank 4: 24825.91 + rank 5: 24825.92 + rank 6: 24825.92 + rank 7: 24825.91 + rank 8: 24826.02 + rank 9: 24826.03 + rank 10: 24826.07 + rank 11: 24826.06 + rank 12: 24826.03 + rank 13: 24825.99 + rank 14: 24826.04 + rank 15: 24826.00 + rank 16: 24825.18 + rank 17: 24825.18 + rank 18: 24825.18 + rank 19: 24825.17 + rank 20: 24825.16 + rank 21: 24825.15 + rank 22: 24826.32 + rank 23: 24825.17 + rank 24: 24825.47 + rank 25: 24825.46 + rank 26: 24826.21 + rank 27: 24825.49 + rank 28: 24825.41 + rank 29: 24825.47 + rank 30: 24825.51 + rank 31: 24825.43 + rank 32: 24826.01 + rank 33: 24825.99 + rank 34: 24825.82 + rank 35: 24826.03 + rank 36: 24825.97 + rank 37: 24825.95 + rank 38: 24825.80 + rank 39: 24825.65 + rank 40: 24825.96 + rank 41: 24825.93 + rank 42: 24825.93 + rank 43: 24825.95 + rank 44: 24825.92 + rank 45: 24825.93 + rank 46: 24825.93 + rank 47: 24825.92 + rank 48: 24825.79 + rank 49: 24825.81 + rank 50: 24825.80 + rank 51: 24825.78 + rank 52: 24825.89 + rank 53: 24825.90 + rank 54: 24825.83 + rank 55: 24825.87 + rank 56: 24826.17 + rank 57: 24826.19 + rank 58: 24826.19 + rank 59: 24826.23 + rank 60: 24826.18 + rank 61: 24826.16 + rank 62: 24826.17 + rank 63: 24826.16 + [2026-10-14 14:53:44] iteration 1161/ 2384 | consumed samples: 37152 | elapsed time per iteration (ms): 231760.0 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 72390.3 | valid tokens/s: 72390.3 | physical tokens/s: 73091.6 | loss tokens/s: 56204.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.410467E-01 | lm-global-supervised-tokens: 1.302605E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.361528E-01 | lm-token-buffer-mib: 2.352966E+02 | global_load_balancing_loss: 9.994885E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11180.681 | params rms: 0.060540 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:44:02.419338 | finish at 2026-10-17 21:37:47 + [2026-10-14 14:57:31] iteration 1162/ 2384 | consumed samples: 37184 | elapsed time per iteration (ms): 226304.3 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 74135.6 | valid tokens/s: 74135.6 | physical tokens/s: 74673.0 | loss tokens/s: 55611.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.707312E-01 | lm-global-supervised-tokens: 1.258503E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.982325E-02 | lm-token-buffer-mib: 2.343383E+02 | global_load_balancing_loss: 9.979420E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11180.647 | params rms: 0.060540 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:49:03.836913 | finish at 2026-10-17 19:46:34 + [2026-10-14 15:01:14] iteration 1163/ 2384 | consumed samples: 37216 | elapsed time per iteration (ms): 223779.0 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 74972.1 | valid tokens/s: 74972.1 | physical tokens/s: 75871.2 | loss tokens/s: 52704.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.741372E-01 | lm-global-supervised-tokens: 1.179411E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.040522E-02 | lm-token-buffer-mib: 2.344381E+02 | global_load_balancing_loss: 9.991575E-01 | loss scale: 1.0 | grad norm: 0.422 | params norm: 11180.612 | params rms: 0.060539 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:53:54.102578 | finish at 2026-10-17 18:55:08 + [2026-10-14 15:04:59] iteration 1164/ 2384 | consumed samples: 37248 | elapsed time per iteration (ms): 224408.8 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.74% | FLOPs estimator: workload-aware | logical tokens/s: 74761.7 | valid tokens/s: 74761.7 | physical tokens/s: 75962.5 | loss tokens/s: 59612.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.190380E-01 | lm-global-supervised-tokens: 1.337761E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.573770E-02 | lm-token-buffer-mib: 2.359133E+02 | global_load_balancing_loss: 1.000873E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11180.580 | params rms: 0.060539 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:02:58.790836 | finish at 2026-10-17 19:07:58 + [2026-10-14 15:08:43] iteration 1165/ 2384 | consumed samples: 37280 | elapsed time per iteration (ms): 223906.0 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 74929.7 | valid tokens/s: 74929.7 | physical tokens/s: 75648.9 | loss tokens/s: 53222.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.490978E-01 | lm-global-supervised-tokens: 1.191690E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.070821E-02 | lm-token-buffer-mib: 2.328503E+02 | global_load_balancing_loss: 1.002058E+00 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11180.549 | params rms: 0.060539 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:49:01.444684 | finish at 2026-10-17 18:57:44 + [2026-10-14 15:12:26] iteration 1166/ 2384 | consumed samples: 37312 | elapsed time per iteration (ms): 223209.8 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.48% | FLOPs estimator: workload-aware | logical tokens/s: 75163.3 | valid tokens/s: 75163.3 | physical tokens/s: 75752.7 | loss tokens/s: 56790.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.037917E-01 | lm-global-supervised-tokens: 1.267621E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.610540E-02 | lm-token-buffer-mib: 2.346744E+02 | global_load_balancing_loss: 1.002737E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11180.514 | params rms: 0.060539 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:31:09.502140 | finish at 2026-10-17 18:43:35 + [2026-10-14 15:16:08] iteration 1167/ 2384 | consumed samples: 37344 | elapsed time per iteration (ms): 222449.6 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 75420.3 | valid tokens/s: 75420.2 | physical tokens/s: 75757.5 | loss tokens/s: 57997.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.123117E-01 | lm-global-supervised-tokens: 1.290162E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.718651E-02 | lm-token-buffer-mib: 2.329358E+02 | global_load_balancing_loss: 1.001419E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11180.474 | params rms: 0.060539 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:12:01.157664 | finish at 2026-10-17 18:28:09 + [2026-10-14 15:19:55] iteration 1168/ 2384 | consumed samples: 37376 | elapsed time per iteration (ms): 226788.6 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 73977.3 | valid tokens/s: 73977.3 | physical tokens/s: 74812.4 | loss tokens/s: 54891.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.437182E-01 | lm-global-supervised-tokens: 1.244882E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.623556E-02 | lm-token-buffer-mib: 2.343251E+02 | global_load_balancing_loss: 1.000635E+00 | loss scale: 1.0 | grad norm: 0.297 | params norm: 11180.436 | params rms: 0.060538 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:36:14.914658 | finish at 2026-10-17 19:56:10 + [2026-10-14 15:23:40] iteration 1169/ 2384 | consumed samples: 37408 | elapsed time per iteration (ms): 224571.3 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.09% | FLOPs estimator: workload-aware | logical tokens/s: 74707.7 | valid tokens/s: 74707.7 | physical tokens/s: 76088.5 | loss tokens/s: 55845.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.125821E-01 | lm-global-supervised-tokens: 1.254136E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.070070E-02 | lm-token-buffer-mib: 2.335924E+02 | global_load_balancing_loss: 9.999418E-01 | loss scale: 1.0 | grad norm: 0.226 | params norm: 11180.401 | params rms: 0.060538 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:47:34.069862 | finish at 2026-10-17 19:11:14 + [2026-10-14 15:27:44] iteration 1170/ 2384 | consumed samples: 37440 | elapsed time per iteration (ms): 244740.5 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.28% | FLOPs estimator: workload-aware | logical tokens/s: 68551.0 | valid tokens/s: 68551.0 | physical tokens/s: 68933.3 | loss tokens/s: 51302.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.044866E-01 | lm-global-supervised-tokens: 1.255584E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.249111E-02 | lm-token-buffer-mib: 2.338684E+02 | global_load_balancing_loss: 1.001531E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11180.367 | params rms: 0.060538 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:31:55.017330 | finish at 2026-10-18 01:59:39 + [2026-10-14 15:31:29] iteration 1171/ 2384 | consumed samples: 37472 | elapsed time per iteration (ms): 224423.0 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 74757.1 | valid tokens/s: 74757.1 | physical tokens/s: 75765.8 | loss tokens/s: 55946.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.623828E-01 | lm-global-supervised-tokens: 1.255568E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.607698E-02 | lm-token-buffer-mib: 2.357033E+02 | global_load_balancing_loss: 9.987714E-01 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11180.330 | params rms: 0.060538 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:37:05.144523 | finish at 2026-10-17 19:08:34 + [2026-10-14 15:35:11] iteration 1172/ 2384 | consumed samples: 37504 | elapsed time per iteration (ms): 222644.3 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 75354.2 | valid tokens/s: 75354.2 | physical tokens/s: 75740.4 | loss tokens/s: 56526.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.382443E-01 | lm-global-supervised-tokens: 1.258538E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.350089E-02 | lm-token-buffer-mib: 2.326138E+02 | global_load_balancing_loss: 9.986102E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11180.294 | params rms: 0.060538 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:57:24.900538 | finish at 2026-10-17 18:32:36 + [2026-10-14 15:38:54] iteration 1173/ 2384 | consumed samples: 37536 | elapsed time per iteration (ms): 222060.4 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 75552.4 | valid tokens/s: 75552.4 | physical tokens/s: 76321.7 | loss tokens/s: 56278.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.769115E-01 | lm-global-supervised-tokens: 1.249716E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.406098E-02 | lm-token-buffer-mib: 2.342860E+02 | global_load_balancing_loss: 1.000258E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11180.258 | params rms: 0.060537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:41:55.120158 | finish at 2026-10-17 18:20:49 + [2026-10-14 15:42:35] iteration 1174/ 2384 | consumed samples: 37568 | elapsed time per iteration (ms): 221469.3 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 75754.1 | valid tokens/s: 75754.1 | physical tokens/s: 76165.6 | loss tokens/s: 55755.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.560747E-01 | lm-global-supervised-tokens: 1.234821E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.477192E-02 | lm-token-buffer-mib: 2.327155E+02 | global_load_balancing_loss: 1.001511E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11180.224 | params rms: 0.060537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:26:17.907274 | finish at 2026-10-17 18:08:53 + [2026-10-14 15:46:16] iteration 1175/ 2384 | consumed samples: 37600 | elapsed time per iteration (ms): 220617.6 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 76046.5 | valid tokens/s: 76046.5 | physical tokens/s: 76623.0 | loss tokens/s: 58520.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.712650E-01 | lm-global-supervised-tokens: 1.291059E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.608872E-02 | lm-token-buffer-mib: 2.339614E+02 | global_load_balancing_loss: 1.001041E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11180.191 | params rms: 0.060537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:05:26.678933 | finish at 2026-10-17 17:51:42 + [2026-10-14 15:49:59] iteration 1176/ 2384 | consumed samples: 37632 | elapsed time per iteration (ms): 223706.7 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 74996.4 | valid tokens/s: 74996.4 | physical tokens/s: 75702.6 | loss tokens/s: 55083.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.348997E-01 | lm-global-supervised-tokens: 1.232247E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.815228E-02 | lm-token-buffer-mib: 2.348161E+02 | global_load_balancing_loss: 1.000576E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11180.158 | params rms: 0.060537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:03:57.653095 | finish at 2026-10-17 18:53:57 + [2026-10-14 15:53:40] iteration 1177/ 2384 | consumed samples: 37664 | elapsed time per iteration (ms): 220502.6 | throughput per GPU (TFLOP/s/GPU): 25.3 | MFU 8.10% | FLOPs estimator: workload-aware | logical tokens/s: 76085.9 | valid tokens/s: 76085.9 | physical tokens/s: 76680.4 | loss tokens/s: 56848.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.328022E-01 | lm-global-supervised-tokens: 1.253531E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.816325E-02 | lm-token-buffer-mib: 2.337222E+02 | global_load_balancing_loss: 1.001055E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11180.123 | params rms: 0.060537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:55:46.690807 | finish at 2026-10-17 17:49:27 + [2026-10-14 15:57:27] iteration 1178/ 2384 | consumed samples: 37696 | elapsed time per iteration (ms): 226701.0 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 74005.8 | valid tokens/s: 74005.8 | physical tokens/s: 74938.9 | loss tokens/s: 51818.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.026974E-01 | lm-global-supervised-tokens: 1.174727E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.251570E-02 | lm-token-buffer-mib: 2.371241E+02 | global_load_balancing_loss: 1.001294E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11180.088 | params rms: 0.060537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:56:41.452550 | finish at 2026-10-17 19:54:08 + [2026-10-14 16:01:09] iteration 1179/ 2384 | consumed samples: 37728 | elapsed time per iteration (ms): 222697.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 75336.4 | valid tokens/s: 75336.4 | physical tokens/s: 76442.3 | loss tokens/s: 58980.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.407032E-01 | lm-global-supervised-tokens: 1.313470E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.791490E-02 | lm-token-buffer-mib: 2.341912E+02 | global_load_balancing_loss: 1.001554E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11180.050 | params rms: 0.060536 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:32:30.065166 | finish at 2026-10-17 18:33:39 + [2026-10-14 16:05:13] iteration 1180/ 2384 | consumed samples: 37760 | elapsed time per iteration (ms): 243828.0 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 68807.4 | valid tokens/s: 68807.4 | physical tokens/s: 69340.4 | loss tokens/s: 50389.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.165939E-01 | lm-global-supervised-tokens: 1.228635E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.755871E-02 | lm-token-buffer-mib: 2.346964E+02 | global_load_balancing_loss: 1.001030E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11180.014 | params rms: 0.060536 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:32:48.929306 | finish at 2026-10-18 01:38:02 + [2026-10-14 16:08:52] iteration 1181/ 2384 | consumed samples: 37792 | elapsed time per iteration (ms): 218948.5 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 76626.1 | valid tokens/s: 76626.1 | physical tokens/s: 76956.6 | loss tokens/s: 53191.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.636237E-01 | lm-global-supervised-tokens: 1.164628E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.669604E-02 | lm-token-buffer-mib: 2.324829E+02 | global_load_balancing_loss: 1.001598E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11179.981 | params rms: 0.060536 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:09:55.067773 | finish at 2026-10-17 17:18:47 + [2026-10-14 16:12:36] iteration 1182/ 2384 | consumed samples: 37824 | elapsed time per iteration (ms): 224012.1 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 74894.2 | valid tokens/s: 74894.2 | physical tokens/s: 76052.8 | loss tokens/s: 60911.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.086634E-01 | lm-global-supervised-tokens: 1.364487E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.639668E-02 | lm-token-buffer-mib: 2.375621E+02 | global_load_balancing_loss: 1.001014E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11179.948 | params rms: 0.060536 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:47:42.486862 | finish at 2026-10-17 19:00:19 + [2026-10-14 16:16:19] iteration 1183/ 2384 | consumed samples: 37856 | elapsed time per iteration (ms): 222521.6 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 75395.6 | valid tokens/s: 75395.6 | physical tokens/s: 75946.1 | loss tokens/s: 55332.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.625188E-01 | lm-global-supervised-tokens: 1.231274E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.620486E-02 | lm-token-buffer-mib: 2.326998E+02 | global_load_balancing_loss: 9.992399E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11179.914 | params rms: 0.060536 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:14:08.459076 | finish at 2026-10-17 18:30:27 + [2026-10-14 16:20:01] iteration 1184/ 2384 | consumed samples: 37888 | elapsed time per iteration (ms): 222344.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 75455.9 | valid tokens/s: 75455.9 | physical tokens/s: 76439.1 | loss tokens/s: 60081.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.819687E-01 | lm-global-supervised-tokens: 1.335866E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.379441E-02 | lm-token-buffer-mib: 2.341783E+02 | global_load_balancing_loss: 1.000514E+00 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11179.880 | params rms: 0.060535 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:06:52.823009 | finish at 2026-10-17 18:26:54 + [2026-10-14 16:23:46] iteration 1185/ 2384 | consumed samples: 37920 | elapsed time per iteration (ms): 225120.4 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 74525.3 | valid tokens/s: 74525.3 | physical tokens/s: 74748.4 | loss tokens/s: 52983.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.900678E-01 | lm-global-supervised-tokens: 1.192772E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.744428E-02 | lm-token-buffer-mib: 2.325406E+02 | global_load_balancing_loss: 1.000651E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11179.845 | params rms: 0.060535 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:58:39.337531 | finish at 2026-10-17 19:22:25 + [2026-10-14 16:27:31] iteration 1186/ 2384 | consumed samples: 37952 | elapsed time per iteration (ms): 225156.7 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 74513.4 | valid tokens/s: 74513.4 | physical tokens/s: 74753.7 | loss tokens/s: 58089.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.518036E-01 | lm-global-supervised-tokens: 1.307912E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.722190E-02 | lm-token-buffer-mib: 2.328226E+02 | global_load_balancing_loss: 9.997135E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11179.813 | params rms: 0.060535 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:55:37.730474 | finish at 2026-10-17 19:23:09 + [2026-10-14 16:31:16] iteration 1187/ 2384 | consumed samples: 37984 | elapsed time per iteration (ms): 224858.9 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.58% | FLOPs estimator: workload-aware | logical tokens/s: 74612.0 | valid tokens/s: 74612.0 | physical tokens/s: 75520.4 | loss tokens/s: 54051.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.828751E-01 | lm-global-supervised-tokens: 1.215385E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.457452E-02 | lm-token-buffer-mib: 2.360953E+02 | global_load_balancing_loss: 1.000076E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11179.777 | params rms: 0.060535 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:45:56.152471 | finish at 2026-10-17 19:17:12 + [2026-10-14 16:34:59] iteration 1188/ 2384 | consumed samples: 38016 | elapsed time per iteration (ms): 222915.9 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 75262.5 | valid tokens/s: 75262.5 | physical tokens/s: 76165.8 | loss tokens/s: 59494.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.134053E-01 | lm-global-supervised-tokens: 1.326218E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.404674E-02 | lm-token-buffer-mib: 2.333572E+02 | global_load_balancing_loss: 1.000155E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11179.738 | params rms: 0.060535 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:03:27.402133 | finish at 2026-10-17 18:38:26 + [2026-10-14 16:38:46] iteration 1189/ 2384 | consumed samples: 38048 | elapsed time per iteration (ms): 226980.0 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 73914.9 | valid tokens/s: 73914.9 | physical tokens/s: 74260.6 | loss tokens/s: 56633.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.073934E-01 | lm-global-supervised-tokens: 1.285467E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.724665E-02 | lm-token-buffer-mib: 2.335326E+02 | global_load_balancing_loss: 1.002155E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11179.701 | params rms: 0.060534 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:20:41.089481 | finish at 2026-10-17 19:59:27 + [2026-10-14 16:42:48] iteration 1190/ 2384 | consumed samples: 38080 | elapsed time per iteration (ms): 241987.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 69331.0 | valid tokens/s: 69331.0 | physical tokens/s: 69631.8 | loss tokens/s: 50832.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.086575E-01 | lm-global-supervised-tokens: 1.230079E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.455717E-02 | lm-token-buffer-mib: 2.328443E+02 | global_load_balancing_loss: 1.001113E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11179.663 | params rms: 0.060534 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:15:32.470870 | finish at 2026-10-18 00:58:20 + [2026-10-14 16:46:30] iteration 1191/ 2384 | consumed samples: 38112 | elapsed time per iteration (ms): 221965.7 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 75584.5 | valid tokens/s: 75584.5 | physical tokens/s: 76444.8 | loss tokens/s: 54973.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.240275E-01 | lm-global-supervised-tokens: 1.220226E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.365102E-02 | lm-token-buffer-mib: 2.339290E+02 | global_load_balancing_loss: 1.000714E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11179.627 | params rms: 0.060534 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:33:25.065203 | finish at 2026-10-17 18:19:55 + [2026-10-14 16:50:13] iteration 1192/ 2384 | consumed samples: 38144 | elapsed time per iteration (ms): 223387.4 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 75103.6 | valid tokens/s: 75103.6 | physical tokens/s: 75648.6 | loss tokens/s: 59091.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.197039E-01 | lm-global-supervised-tokens: 1.320032E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.906828E-02 | lm-token-buffer-mib: 2.336006E+02 | global_load_balancing_loss: 1.000874E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11179.590 | params rms: 0.060534 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:57:57.726698 | finish at 2026-10-17 18:48:11 + [2026-10-14 16:53:56] iteration 1193/ 2384 | consumed samples: 38176 | elapsed time per iteration (ms): 223205.0 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 75164.9 | valid tokens/s: 75164.9 | physical tokens/s: 76029.5 | loss tokens/s: 60795.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.604229E-01 | lm-global-supervised-tokens: 1.356978E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.276848E-02 | lm-token-buffer-mib: 2.338818E+02 | global_load_balancing_loss: 1.000587E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11179.555 | params rms: 0.060534 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:50:37.188984 | finish at 2026-10-17 18:44:34 + [2026-10-14 16:57:40] iteration 1194/ 2384 | consumed samples: 38208 | elapsed time per iteration (ms): 223809.1 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 74961.9 | valid tokens/s: 74961.9 | physical tokens/s: 75640.2 | loss tokens/s: 55616.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.003845E-01 | lm-global-supervised-tokens: 1.244750E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.664330E-02 | lm-token-buffer-mib: 2.356501E+02 | global_load_balancing_loss: 1.001276E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11179.525 | params rms: 0.060533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:58:52.853580 | finish at 2026-10-17 18:56:33 + [2026-10-14 17:01:26] iteration 1195/ 2384 | consumed samples: 38240 | elapsed time per iteration (ms): 225899.5 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.09% | FLOPs estimator: workload-aware | logical tokens/s: 74268.3 | valid tokens/s: 74268.3 | physical tokens/s: 74752.4 | loss tokens/s: 56139.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 9.147135E-01 | lm-global-supervised-tokens: 1.268189E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.515220E-02 | lm-token-buffer-mib: 2.329504E+02 | global_load_balancing_loss: 9.997327E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11179.498 | params rms: 0.060533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:36:34.530036 | finish at 2026-10-17 19:38:01 + [2026-10-14 17:05:07] iteration 1196/ 2384 | consumed samples: 38272 | elapsed time per iteration (ms): 221276.8 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.68% | FLOPs estimator: workload-aware | logical tokens/s: 75819.8 | valid tokens/s: 75819.8 | physical tokens/s: 76218.6 | loss tokens/s: 51460.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.123810E-01 | lm-global-supervised-tokens: 1.138706E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.826312E-02 | lm-token-buffer-mib: 2.327356E+02 | global_load_balancing_loss: 9.986256E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11179.468 | params rms: 0.060533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:01:16.813943 | finish at 2026-10-17 18:06:24 + [2026-10-14 17:08:51] iteration 1197/ 2384 | consumed samples: 38304 | elapsed time per iteration (ms): 223357.1 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 75113.7 | valid tokens/s: 75113.7 | physical tokens/s: 75863.5 | loss tokens/s: 55703.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.266836E-01 | lm-global-supervised-tokens: 1.244168E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.500621E-02 | lm-token-buffer-mib: 2.363567E+02 | global_load_balancing_loss: 9.995536E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11179.433 | params rms: 0.060533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:38:44.830167 | finish at 2026-10-17 18:47:36 + [2026-10-14 17:12:34] iteration 1198/ 2384 | consumed samples: 38336 | elapsed time per iteration (ms): 222981.1 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 75240.5 | valid tokens/s: 75240.5 | physical tokens/s: 75657.6 | loss tokens/s: 58165.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.267778E-01 | lm-global-supervised-tokens: 1.296972E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.420456E-02 | lm-token-buffer-mib: 2.327872E+02 | global_load_balancing_loss: 9.998555E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11179.401 | params rms: 0.060533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:27:35.529559 | finish at 2026-10-17 18:40:09 + [2026-10-14 17:16:20] iteration 1199/ 2384 | consumed samples: 38368 | elapsed time per iteration (ms): 225970.3 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 74245.1 | valid tokens/s: 74245.1 | physical tokens/s: 75271.6 | loss tokens/s: 54879.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.309614E-01 | lm-global-supervised-tokens: 1.240120E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.946525E-02 | lm-token-buffer-mib: 2.353001E+02 | global_load_balancing_loss: 1.000136E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11179.371 | params rms: 0.060533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:22:54.785957 | finish at 2026-10-17 19:39:15 + [2026-10-14 17:20:27] iteration 1200/ 2384 | consumed samples: 38400 | elapsed time per iteration (ms): 246812.3 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 67975.4 | valid tokens/s: 67975.4 | physical tokens/s: 68604.7 | loss tokens/s: 51951.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.417962E-01 | lm-global-supervised-tokens: 1.282229E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.957560E-02 | lm-token-buffer-mib: 2.354019E+02 | global_load_balancing_loss: 1.001145E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11179.342 | params rms: 0.060533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:10:25.749611 | finish at 2026-10-18 02:30:52 +times across ranks (ms): + save-checkpoint: + rank 0: 24728.38 + rank 1: 24728.37 + rank 2: 24728.43 + rank 3: 24728.42 + rank 4: 24728.40 + rank 5: 24728.40 + rank 6: 24728.41 + rank 7: 24728.40 + rank 8: 24727.82 + rank 9: 24727.85 + rank 10: 24727.86 + rank 11: 24727.84 + rank 12: 24727.82 + rank 13: 24727.82 + rank 14: 24727.83 + rank 15: 24727.83 + rank 16: 24728.56 + rank 17: 24728.56 + rank 18: 24728.54 + rank 19: 24728.55 + rank 20: 24728.53 + rank 21: 24728.52 + rank 22: 24728.60 + rank 23: 24728.52 + rank 24: 24728.53 + rank 25: 24728.50 + rank 26: 24728.54 + rank 27: 24728.56 + rank 28: 24728.50 + rank 29: 24728.54 + rank 30: 24728.59 + rank 31: 24728.51 + rank 32: 24728.52 + rank 33: 24728.54 + rank 34: 24728.53 + rank 35: 24728.55 + rank 36: 24728.48 + rank 37: 24728.49 + rank 38: 24728.49 + rank 39: 24728.49 + rank 40: 24728.20 + rank 41: 24728.20 + rank 42: 24728.19 + rank 43: 24728.21 + rank 44: 24728.19 + rank 45: 24728.20 + rank 46: 24728.28 + rank 47: 24728.18 + rank 48: 24728.35 + rank 49: 24728.35 + rank 50: 24728.40 + rank 51: 24728.35 + rank 52: 24728.34 + rank 53: 24728.36 + rank 54: 24728.34 + rank 55: 24728.50 + rank 56: 24728.38 + rank 57: 24728.37 + rank 58: 24728.38 + rank 59: 24728.37 + rank 60: 24728.39 + rank 61: 24728.37 + rank 62: 24728.36 + rank 63: 24728.38 + [2026-10-14 17:24:30] iteration 1201/ 2384 | consumed samples: 38432 | elapsed time per iteration (ms): 218134.8 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 76912.0 | valid tokens/s: 76912.0 | physical tokens/s: 77543.2 | loss tokens/s: 58053.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.640007E-01 | lm-global-supervised-tokens: 1.266342E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.409846E-01 | lm-token-buffer-mib: 2.334067E+02 | global_load_balancing_loss: 1.001387E+00 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11179.312 | params rms: 0.060532 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:40:53.515734 | finish at 2026-10-17 17:05:23 + [2026-10-14 17:28:16] iteration 1202/ 2384 | consumed samples: 38464 | elapsed time per iteration (ms): 226564.0 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 74050.5 | valid tokens/s: 74050.5 | physical tokens/s: 74628.1 | loss tokens/s: 53484.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.568495E-01 | lm-global-supervised-tokens: 1.211765E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.406286E-02 | lm-token-buffer-mib: 2.336943E+02 | global_load_balancing_loss: 1.001873E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11179.278 | params rms: 0.060532 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:23:18.638572 | finish at 2026-10-17 19:51:35 + [2026-10-14 17:31:57] iteration 1203/ 2384 | consumed samples: 38496 | elapsed time per iteration (ms): 221166.6 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 75857.8 | valid tokens/s: 75857.8 | physical tokens/s: 76697.3 | loss tokens/s: 59538.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.405604E-01 | lm-global-supervised-tokens: 1.316791E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.395875E-02 | lm-token-buffer-mib: 2.347876E+02 | global_load_balancing_loss: 1.001032E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11179.241 | params rms: 0.060532 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:33:17.696865 | finish at 2026-10-17 18:05:15 + [2026-10-14 17:35:45] iteration 1204/ 2384 | consumed samples: 38528 | elapsed time per iteration (ms): 227453.9 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 73760.9 | valid tokens/s: 73760.9 | physical tokens/s: 74753.6 | loss tokens/s: 56501.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.612304E-01 | lm-global-supervised-tokens: 1.285146E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.387241E-02 | lm-token-buffer-mib: 2.373554E+02 | global_load_balancing_loss: 1.000927E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11179.204 | params rms: 0.060532 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:33:15.638409 | finish at 2026-10-17 20:09:00 + [2026-10-14 17:39:28] iteration 1205/ 2384 | consumed samples: 38560 | elapsed time per iteration (ms): 223360.8 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.04% | FLOPs estimator: workload-aware | logical tokens/s: 75112.5 | valid tokens/s: 75112.5 | physical tokens/s: 75592.5 | loss tokens/s: 55349.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.415148E-01 | lm-global-supervised-tokens: 1.236293E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.327399E-02 | lm-token-buffer-mib: 2.331604E+02 | global_load_balancing_loss: 1.000473E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11179.173 | params rms: 0.060532 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:09:02.440014 | finish at 2026-10-17 18:48:31 + [2026-10-14 17:43:11] iteration 1206/ 2384 | consumed samples: 38592 | elapsed time per iteration (ms): 222970.0 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 75244.1 | valid tokens/s: 75244.1 | physical tokens/s: 76079.0 | loss tokens/s: 54812.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.656787E-01 | lm-global-supervised-tokens: 1.222161E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.849872E-02 | lm-token-buffer-mib: 2.349250E+02 | global_load_balancing_loss: 9.993583E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11179.143 | params rms: 0.060531 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:57:38.608637 | finish at 2026-10-17 18:40:50 + [2026-10-14 17:46:54] iteration 1207/ 2384 | consumed samples: 38624 | elapsed time per iteration (ms): 222690.0 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 75338.7 | valid tokens/s: 75338.7 | physical tokens/s: 76062.5 | loss tokens/s: 56538.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.670893E-01 | lm-global-supervised-tokens: 1.259054E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.156962E-02 | lm-token-buffer-mib: 2.336432E+02 | global_load_balancing_loss: 9.990898E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11179.109 | params rms: 0.060531 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:48:26.183946 | finish at 2026-10-17 18:35:20 + [2026-10-14 17:50:38] iteration 1208/ 2384 | consumed samples: 38656 | elapsed time per iteration (ms): 224233.4 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 74820.3 | valid tokens/s: 74820.3 | physical tokens/s: 75568.1 | loss tokens/s: 60772.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.548364E-01 | lm-global-supervised-tokens: 1.362727E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.464460E-02 | lm-token-buffer-mib: 2.345916E+02 | global_load_balancing_loss: 1.000539E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11179.073 | params rms: 0.060531 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:14:58.446562 | finish at 2026-10-17 19:05:37 + [2026-10-14 17:54:22] iteration 1209/ 2384 | consumed samples: 38688 | elapsed time per iteration (ms): 224140.1 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 74851.3 | valid tokens/s: 74851.3 | physical tokens/s: 76088.4 | loss tokens/s: 57719.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.691324E-01 | lm-global-supervised-tokens: 1.293724E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.978322E-02 | lm-token-buffer-mib: 2.343004E+02 | global_load_balancing_loss: 1.000634E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11179.037 | params rms: 0.060531 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:09:24.580244 | finish at 2026-10-17 19:03:47 + [2026-10-14 17:58:24] iteration 1210/ 2384 | consumed samples: 38720 | elapsed time per iteration (ms): 242179.6 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.02% | FLOPs estimator: workload-aware | logical tokens/s: 69275.8 | valid tokens/s: 69275.8 | physical tokens/s: 69670.2 | loss tokens/s: 47354.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.580021E-01 | lm-global-supervised-tokens: 1.146824E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.753217E-02 | lm-token-buffer-mib: 2.329861E+02 | global_load_balancing_loss: 1.001199E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11179.002 | params rms: 0.060531 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:58:38.908900 | finish at 2026-10-18 00:57:03 + [2026-10-14 18:02:08] iteration 1211/ 2384 | consumed samples: 38752 | elapsed time per iteration (ms): 224073.4 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 74873.7 | valid tokens/s: 74873.7 | physical tokens/s: 75630.9 | loss tokens/s: 59013.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.681439E-01 | lm-global-supervised-tokens: 1.322339E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.668071E-02 | lm-token-buffer-mib: 2.339035E+02 | global_load_balancing_loss: 1.002257E+00 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11178.968 | params rms: 0.060530 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:00:38.073614 | finish at 2026-10-17 19:02:47 + [2026-10-14 18:05:51] iteration 1212/ 2384 | consumed samples: 38784 | elapsed time per iteration (ms): 222635.8 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.19% | FLOPs estimator: workload-aware | logical tokens/s: 75357.2 | valid tokens/s: 75357.2 | physical tokens/s: 76486.7 | loss tokens/s: 56041.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.326868E-01 | lm-global-supervised-tokens: 1.247676E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.930829E-02 | lm-token-buffer-mib: 2.366526E+02 | global_load_balancing_loss: 9.997529E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11178.933 | params rms: 0.060530 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:28:49.119742 | finish at 2026-10-17 18:34:40 + [2026-10-14 18:09:40] iteration 1213/ 2384 | consumed samples: 38816 | elapsed time per iteration (ms): 228862.8 | throughput per GPU (TFLOP/s/GPU): 28.3 | MFU 9.09% | FLOPs estimator: workload-aware | logical tokens/s: 73306.5 | valid tokens/s: 73306.5 | physical tokens/s: 74074.2 | loss tokens/s: 54296.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.531867E-01 | lm-global-supervised-tokens: 1.242637E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.716827E-02 | lm-token-buffer-mib: 2.347820E+02 | global_load_balancing_loss: 9.992374E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11178.896 | params rms: 0.060530 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:26:38.327216 | finish at 2026-10-17 20:36:18 + [2026-10-14 18:13:23] iteration 1214/ 2384 | consumed samples: 38848 | elapsed time per iteration (ms): 222999.7 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 75234.2 | valid tokens/s: 75234.2 | physical tokens/s: 75852.4 | loss tokens/s: 59551.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.087375E-01 | lm-global-supervised-tokens: 1.327991E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.309246E-02 | lm-token-buffer-mib: 2.328928E+02 | global_load_balancing_loss: 9.988249E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11178.855 | params rms: 0.060530 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:28:29.639039 | finish at 2026-10-17 18:41:53 + [2026-10-14 18:17:09] iteration 1215/ 2384 | consumed samples: 38880 | elapsed time per iteration (ms): 225694.4 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 74335.8 | valid tokens/s: 74335.8 | physical tokens/s: 75126.3 | loss tokens/s: 54672.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.798705E-01 | lm-global-supervised-tokens: 1.233936E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.477212E-02 | lm-token-buffer-mib: 2.345316E+02 | global_load_balancing_loss: 1.000020E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11178.815 | params rms: 0.060530 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:17:16.791032 | finish at 2026-10-17 19:34:25 + [2026-10-14 18:20:49] iteration 1216/ 2384 | consumed samples: 38912 | elapsed time per iteration (ms): 220777.8 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 75991.3 | valid tokens/s: 75991.3 | physical tokens/s: 76424.8 | loss tokens/s: 54068.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.225710E-01 | lm-global-supervised-tokens: 1.193701E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.310190E-02 | lm-token-buffer-mib: 2.338266E+02 | global_load_balancing_loss: 1.000463E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11178.772 | params rms: 0.060529 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:37:48.487206 | finish at 2026-10-17 17:58:38 + [2026-10-14 18:24:30] iteration 1217/ 2384 | consumed samples: 38944 | elapsed time per iteration (ms): 220197.6 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 76191.5 | valid tokens/s: 76191.5 | physical tokens/s: 76463.1 | loss tokens/s: 58867.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.584066E-01 | lm-global-supervised-tokens: 1.296255E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.880610E-02 | lm-token-buffer-mib: 2.330199E+02 | global_load_balancing_loss: 1.000412E+00 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11178.730 | params rms: 0.060529 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:22:50.647760 | finish at 2026-10-17 17:47:20 + [2026-10-14 18:28:14] iteration 1218/ 2384 | consumed samples: 38976 | elapsed time per iteration (ms): 224740.9 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 74651.3 | valid tokens/s: 74651.3 | physical tokens/s: 75556.7 | loss tokens/s: 59865.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.003392E-01 | lm-global-supervised-tokens: 1.345422E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.456479E-02 | lm-token-buffer-mib: 2.340899E+02 | global_load_balancing_loss: 1.000348E+00 | loss scale: 1.0 | grad norm: 0.261 | params norm: 11178.687 | params rms: 0.060529 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:47:27.886666 | finish at 2026-10-17 19:15:42 + [2026-10-14 18:31:59] iteration 1219/ 2384 | consumed samples: 39008 | elapsed time per iteration (ms): 224628.0 | throughput per GPU (TFLOP/s/GPU): 18.1 | MFU 5.81% | FLOPs estimator: workload-aware | logical tokens/s: 74688.9 | valid tokens/s: 74688.9 | physical tokens/s: 76577.3 | loss tokens/s: 60726.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.449032E-01 | lm-global-supervised-tokens: 1.364075E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.964252E-02 | lm-token-buffer-mib: 2.350914E+02 | global_load_balancing_loss: 1.002329E+00 | loss scale: 1.0 | grad norm: 1.224 | params norm: 11178.644 | params rms: 0.060529 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:41:31.570861 | finish at 2026-10-17 19:13:31 + [2026-10-14 18:36:03] iteration 1220/ 2384 | consumed samples: 39040 | elapsed time per iteration (ms): 244074.9 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 68737.8 | valid tokens/s: 68737.8 | physical tokens/s: 69320.4 | loss tokens/s: 50931.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.153302E-01 | lm-global-supervised-tokens: 1.243110E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.759650E-02 | lm-token-buffer-mib: 2.341898E+02 | global_load_balancing_loss: 1.001359E+00 | loss scale: 1.0 | grad norm: 0.493 | params norm: 11178.599 | params rms: 0.060528 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:55:03.137152 | finish at 2026-10-18 01:31:06 + [2026-10-14 18:39:46] iteration 1221/ 2384 | consumed samples: 39072 | elapsed time per iteration (ms): 223286.2 | throughput per GPU (TFLOP/s/GPU): 26.0 | MFU 8.32% | FLOPs estimator: workload-aware | logical tokens/s: 75137.6 | valid tokens/s: 75137.6 | physical tokens/s: 75559.6 | loss tokens/s: 53951.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.763067E-01 | lm-global-supervised-tokens: 1.204666E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.366898E-02 | lm-token-buffer-mib: 2.330252E+02 | global_load_balancing_loss: 1.002579E+00 | loss scale: 1.0 | grad norm: 0.202 | params norm: 11178.557 | params rms: 0.060528 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:08:01.838454 | finish at 2026-10-17 18:47:48 + [2026-10-14 18:43:25] iteration 1222/ 2384 | consumed samples: 39104 | elapsed time per iteration (ms): 218826.0 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 76669.1 | valid tokens/s: 76669.0 | physical tokens/s: 77526.4 | loss tokens/s: 60024.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.708276E-01 | lm-global-supervised-tokens: 1.313499E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.510650E-02 | lm-token-buffer-mib: 2.334317E+02 | global_load_balancing_loss: 1.000172E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11178.517 | params rms: 0.060528 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:37:55.865440 | finish at 2026-10-17 17:21:21 + [2026-10-14 18:47:07] iteration 1223/ 2384 | consumed samples: 39136 | elapsed time per iteration (ms): 221835.6 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 75629.0 | valid tokens/s: 75629.0 | physical tokens/s: 76370.5 | loss tokens/s: 58004.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.082396E-01 | lm-global-supervised-tokens: 1.286735E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.928480E-02 | lm-token-buffer-mib: 2.347831E+02 | global_load_balancing_loss: 1.001263E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11178.479 | params rms: 0.060528 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:32:31.159440 | finish at 2026-10-17 18:19:38 + [2026-10-14 18:50:56] iteration 1224/ 2384 | consumed samples: 39168 | elapsed time per iteration (ms): 229363.4 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 73146.7 | valid tokens/s: 73146.7 | physical tokens/s: 74400.6 | loss tokens/s: 56398.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.144387E-01 | lm-global-supervised-tokens: 1.293569E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.503353E-02 | lm-token-buffer-mib: 2.378778E+02 | global_load_balancing_loss: 1.000342E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11178.443 | params rms: 0.060528 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:54:21.520748 | finish at 2026-10-17 20:45:18 + [2026-10-14 18:54:40] iteration 1225/ 2384 | consumed samples: 39200 | elapsed time per iteration (ms): 223266.1 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 75144.4 | valid tokens/s: 75144.4 | physical tokens/s: 76207.7 | loss tokens/s: 55793.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.298122E-01 | lm-global-supervised-tokens: 1.245688E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.869614E-02 | lm-token-buffer-mib: 2.348870E+02 | global_load_balancing_loss: 1.000154E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11178.406 | params rms: 0.060527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:52:45.378317 | finish at 2026-10-17 18:47:25 + [2026-10-14 18:58:25] iteration 1226/ 2384 | consumed samples: 39232 | elapsed time per iteration (ms): 225274.8 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 74474.2 | valid tokens/s: 74474.2 | physical tokens/s: 76234.2 | loss tokens/s: 57021.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.757579E-01 | lm-global-supervised-tokens: 1.284551E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.821430E-02 | lm-token-buffer-mib: 2.380683E+02 | global_load_balancing_loss: 9.979057E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11178.370 | params rms: 0.060527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:27:48.169328 | finish at 2026-10-17 19:26:13 + [2026-10-14 19:02:09] iteration 1227/ 2384 | consumed samples: 39264 | elapsed time per iteration (ms): 224031.2 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 74887.8 | valid tokens/s: 74887.8 | physical tokens/s: 75727.5 | loss tokens/s: 56939.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.241657E-01 | lm-global-supervised-tokens: 1.275615E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.488497E-02 | lm-token-buffer-mib: 2.350579E+02 | global_load_balancing_loss: 1.000954E+00 | loss scale: 1.0 | grad norm: 0.180 | params norm: 11178.338 | params rms: 0.060527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:00:04.047841 | finish at 2026-10-17 19:02:13 + [2026-10-14 19:05:52] iteration 1228/ 2384 | consumed samples: 39296 | elapsed time per iteration (ms): 223314.7 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 75128.0 | valid tokens/s: 75128.0 | physical tokens/s: 75575.2 | loss tokens/s: 55051.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.340948E-01 | lm-global-supervised-tokens: 1.229376E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.325000E-02 | lm-token-buffer-mib: 2.332560E+02 | global_load_balancing_loss: 9.981987E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11178.309 | params rms: 0.060527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:42:31.837169 | finish at 2026-10-17 18:48:24 + [2026-10-14 19:09:33] iteration 1229/ 2384 | consumed samples: 39328 | elapsed time per iteration (ms): 220302.4 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 76155.4 | valid tokens/s: 76155.4 | physical tokens/s: 76668.5 | loss tokens/s: 57754.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.633137E-01 | lm-global-supervised-tokens: 1.272345E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.397380E-02 | lm-token-buffer-mib: 2.336131E+02 | global_load_balancing_loss: 9.982369E-01 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11178.280 | params rms: 0.060527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:40:49.217606 | finish at 2026-10-17 17:50:22 + [2026-10-14 19:13:36] iteration 1230/ 2384 | consumed samples: 39360 | elapsed time per iteration (ms): 243431.6 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 68919.5 | valid tokens/s: 68919.5 | physical tokens/s: 69576.1 | loss tokens/s: 45641.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.696827E-01 | lm-global-supervised-tokens: 1.111053E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.584997E-02 | lm-token-buffer-mib: 2.326959E+02 | global_load_balancing_loss: 1.001185E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11178.250 | params rms: 0.060527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:02:00.045048 | finish at 2026-10-18 01:15:36 + [2026-10-14 19:17:20] iteration 1231/ 2384 | consumed samples: 39392 | elapsed time per iteration (ms): 223724.4 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.61% | FLOPs estimator: workload-aware | logical tokens/s: 74990.4 | valid tokens/s: 74990.4 | physical tokens/s: 75807.0 | loss tokens/s: 58645.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.854593E-01 | lm-global-supervised-tokens: 1.312051E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.480871E-02 | lm-token-buffer-mib: 2.329994E+02 | global_load_balancing_loss: 1.001474E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11178.218 | params rms: 0.060526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:39:14.212633 | finish at 2026-10-17 18:56:34 + [2026-10-14 19:21:03] iteration 1232/ 2384 | consumed samples: 39424 | elapsed time per iteration (ms): 223239.9 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 75153.0 | valid tokens/s: 75153.0 | physical tokens/s: 75438.6 | loss tokens/s: 53249.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.848588E-01 | lm-global-supervised-tokens: 1.188732E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.749833E-02 | lm-token-buffer-mib: 2.330574E+02 | global_load_balancing_loss: 1.004338E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11178.187 | params rms: 0.060526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:26:12.339386 | finish at 2026-10-17 18:47:15 + [2026-10-14 19:24:46] iteration 1233/ 2384 | consumed samples: 39456 | elapsed time per iteration (ms): 223075.1 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75208.8 | valid tokens/s: 75208.8 | physical tokens/s: 75900.0 | loss tokens/s: 55636.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.963446E-01 | lm-global-supervised-tokens: 1.241121E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.752227E-02 | lm-token-buffer-mib: 2.348080E+02 | global_load_balancing_loss: 1.000557E+00 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11178.155 | params rms: 0.060526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:19:19.390367 | finish at 2026-10-17 18:44:05 + [2026-10-14 19:28:30] iteration 1234/ 2384 | consumed samples: 39488 | elapsed time per iteration (ms): 223463.5 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.98% | FLOPs estimator: workload-aware | logical tokens/s: 75077.9 | valid tokens/s: 75077.9 | physical tokens/s: 75567.6 | loss tokens/s: 54725.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.133107E-01 | lm-global-supervised-tokens: 1.222923E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.113830E-02 | lm-token-buffer-mib: 2.343388E+02 | global_load_balancing_loss: 9.994747E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11178.125 | params rms: 0.060526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:23:02.980335 | finish at 2026-10-17 18:51:33 + [2026-10-14 19:32:11] iteration 1235/ 2384 | consumed samples: 39520 | elapsed time per iteration (ms): 221945.1 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.39% | FLOPs estimator: workload-aware | logical tokens/s: 75591.4 | valid tokens/s: 75591.4 | physical tokens/s: 76125.5 | loss tokens/s: 54915.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.620901E-01 | lm-global-supervised-tokens: 1.218822E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.176031E-01 | lm-token-buffer-mib: 2.331589E+02 | global_load_balancing_loss: 1.000716E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11178.093 | params rms: 0.060526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:50:14.866834 | finish at 2026-10-17 18:22:26 + [2026-10-14 19:35:52] iteration 1236/ 2384 | consumed samples: 39552 | elapsed time per iteration (ms): 220382.9 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 76127.5 | valid tokens/s: 76127.5 | physical tokens/s: 76449.6 | loss tokens/s: 56227.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.051155E-01 | lm-global-supervised-tokens: 1.239160E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.390898E-02 | lm-token-buffer-mib: 2.329289E+02 | global_load_balancing_loss: 9.990638E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11178.059 | params rms: 0.060526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:16:39.530060 | finish at 2026-10-17 17:52:31 + [2026-10-14 19:39:34] iteration 1237/ 2384 | consumed samples: 39584 | elapsed time per iteration (ms): 222440.5 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75423.4 | valid tokens/s: 75423.4 | physical tokens/s: 76353.3 | loss tokens/s: 60378.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.987812E-01 | lm-global-supervised-tokens: 1.343059E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.455125E-02 | lm-token-buffer-mib: 2.354688E+02 | global_load_balancing_loss: 1.000260E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11178.025 | params rms: 0.060525 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:52:19.272120 | finish at 2026-10-17 18:31:54 + [2026-10-14 19:43:17] iteration 1238/ 2384 | consumed samples: 39616 | elapsed time per iteration (ms): 222725.2 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 75326.9 | valid tokens/s: 75326.9 | physical tokens/s: 75740.8 | loss tokens/s: 58107.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.609098E-01 | lm-global-supervised-tokens: 1.294199E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.684203E-02 | lm-token-buffer-mib: 2.331014E+02 | global_load_balancing_loss: 1.000948E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11177.993 | params rms: 0.060525 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:54:03.093610 | finish at 2026-10-17 18:37:20 + [2026-10-14 19:46:59] iteration 1239/ 2384 | consumed samples: 39648 | elapsed time per iteration (ms): 222392.5 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 75439.6 | valid tokens/s: 75439.6 | physical tokens/s: 75996.2 | loss tokens/s: 54437.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.123561E-01 | lm-global-supervised-tokens: 1.210657E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.403229E-02 | lm-token-buffer-mib: 2.329165E+02 | global_load_balancing_loss: 1.001381E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11177.961 | params rms: 0.060525 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:43:59.362183 | finish at 2026-10-17 18:30:59 + [2026-10-14 19:51:03] iteration 1240/ 2384 | consumed samples: 39680 | elapsed time per iteration (ms): 243482.1 | throughput per GPU (TFLOP/s/GPU): 17.8 | MFU 5.72% | FLOPs estimator: workload-aware | logical tokens/s: 68905.3 | valid tokens/s: 68905.3 | physical tokens/s: 69784.3 | loss tokens/s: 52829.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.509335E-01 | lm-global-supervised-tokens: 1.286307E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.692900E-02 | lm-token-buffer-mib: 2.353288E+02 | global_load_balancing_loss: 1.002034E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11177.927 | params rms: 0.060525 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:22:23.465406 | finish at 2026-10-18 01:13:26 +times across ranks (ms): + save-checkpoint: + rank 0: 26141.91 + rank 1: 26141.83 + rank 2: 26141.90 + rank 3: 26141.88 + rank 4: 26141.85 + rank 5: 26141.88 + rank 6: 26141.87 + rank 7: 26141.86 + rank 8: 26141.65 + rank 9: 26141.66 + rank 10: 26141.66 + rank 11: 26141.70 + rank 12: 26141.65 + rank 13: 26141.66 + rank 14: 26141.58 + rank 15: 26141.64 + rank 16: 26141.74 + rank 17: 26141.70 + rank 18: 26141.72 + rank 19: 26141.70 + rank 20: 26141.69 + rank 21: 26141.70 + rank 22: 26141.78 + rank 23: 26141.71 + rank 24: 26141.80 + rank 25: 26141.78 + rank 26: 26141.81 + rank 27: 26141.53 + rank 28: 26141.76 + rank 29: 26142.40 + rank 30: 26141.80 + rank 31: 26142.09 + rank 32: 26141.59 + rank 33: 26141.84 + rank 34: 26141.47 + rank 35: 26141.73 + rank 36: 26141.71 + rank 37: 26141.71 + rank 38: 26141.70 + rank 39: 26141.68 + rank 40: 26142.35 + rank 41: 26142.31 + rank 42: 26142.32 + rank 43: 26142.32 + rank 44: 26142.30 + rank 45: 26142.31 + rank 46: 26142.31 + rank 47: 26142.31 + rank 48: 26141.39 + rank 49: 26141.43 + rank 50: 26141.45 + rank 51: 26141.38 + rank 52: 26141.38 + rank 53: 26141.54 + rank 54: 26141.44 + rank 55: 26141.35 + rank 56: 26141.93 + rank 57: 26141.95 + rank 58: 26141.97 + rank 59: 26141.97 + rank 60: 26141.97 + rank 61: 26141.92 + rank 62: 26141.92 + rank 63: 26141.92 + [2026-10-14 19:55:07] iteration 1241/ 2384 | consumed samples: 39712 | elapsed time per iteration (ms): 218027.8 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 76949.9 | valid tokens/s: 76949.9 | physical tokens/s: 77417.2 | loss tokens/s: 57629.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.358747E-01 | lm-global-supervised-tokens: 1.256472E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.446148E-02 | lm-token-buffer-mib: 2.330683E+02 | global_load_balancing_loss: 1.000296E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11177.892 | params rms: 0.060525 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:13:25.778220 | finish at 2026-10-17 17:08:33 + [2026-10-14 19:58:54] iteration 1242/ 2384 | consumed samples: 39744 | elapsed time per iteration (ms): 226317.9 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 74131.0 | valid tokens/s: 74131.0 | physical tokens/s: 74453.8 | loss tokens/s: 55250.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.087453E-01 | lm-global-supervised-tokens: 1.250420E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.396111E-02 | lm-token-buffer-mib: 2.325428E+02 | global_load_balancing_loss: 1.000054E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11177.862 | params rms: 0.060524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:47:35.091016 | finish at 2026-10-17 19:46:29 + [2026-10-14 20:02:35] iteration 1243/ 2384 | consumed samples: 39776 | elapsed time per iteration (ms): 221190.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 75849.7 | valid tokens/s: 75849.7 | physical tokens/s: 76320.2 | loss tokens/s: 57612.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.499428E-01 | lm-global-supervised-tokens: 1.274336E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.363160E-02 | lm-token-buffer-mib: 2.332837E+02 | global_load_balancing_loss: 1.000478E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11177.831 | params rms: 0.060524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:06:17.904864 | finish at 2026-10-17 18:08:53 + [2026-10-14 20:06:20] iteration 1244/ 2384 | consumed samples: 39808 | elapsed time per iteration (ms): 225063.4 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.75% | FLOPs estimator: workload-aware | logical tokens/s: 74544.3 | valid tokens/s: 74544.3 | physical tokens/s: 75228.3 | loss tokens/s: 59029.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.474210E-01 | lm-global-supervised-tokens: 1.328527E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.458628E-02 | lm-token-buffer-mib: 2.356615E+02 | global_load_balancing_loss: 9.990667E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11177.799 | params rms: 0.060524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:16:12.328491 | finish at 2026-10-17 19:22:32 + [2026-10-14 20:10:01] iteration 1245/ 2384 | consumed samples: 39840 | elapsed time per iteration (ms): 221311.8 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 75807.9 | valid tokens/s: 75807.9 | physical tokens/s: 76756.3 | loss tokens/s: 55343.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.787077E-01 | lm-global-supervised-tokens: 1.224811E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.335188E-02 | lm-token-buffer-mib: 2.355150E+02 | global_load_balancing_loss: 1.000114E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11177.765 | params rms: 0.060524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:01:14.093495 | finish at 2026-10-17 18:11:15 + [2026-10-14 20:13:45] iteration 1246/ 2384 | consumed samples: 39872 | elapsed time per iteration (ms): 223615.1 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75027.2 | valid tokens/s: 75027.2 | physical tokens/s: 75800.3 | loss tokens/s: 53359.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.587502E-01 | lm-global-supervised-tokens: 1.193189E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.019358E-02 | lm-token-buffer-mib: 2.348928E+02 | global_load_balancing_loss: 1.000589E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11177.729 | params rms: 0.060524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:41:13.931601 | finish at 2026-10-17 18:54:59 + [2026-10-14 20:17:30] iteration 1247/ 2384 | consumed samples: 39904 | elapsed time per iteration (ms): 224663.9 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.44% | FLOPs estimator: workload-aware | logical tokens/s: 74676.8 | valid tokens/s: 74676.8 | physical tokens/s: 75173.5 | loss tokens/s: 55686.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.872892E-01 | lm-global-supervised-tokens: 1.251072E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.287392E-02 | lm-token-buffer-mib: 2.341467E+02 | global_load_balancing_loss: 1.001312E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11177.693 | params rms: 0.060524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:57:22.875329 | finish at 2026-10-17 19:14:52 + [2026-10-14 20:21:10] iteration 1248/ 2384 | consumed samples: 39936 | elapsed time per iteration (ms): 220699.3 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 76018.4 | valid tokens/s: 76018.4 | physical tokens/s: 76742.2 | loss tokens/s: 56115.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.658384E-01 | lm-global-supervised-tokens: 1.238457E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.037838E-02 | lm-token-buffer-mib: 2.337769E+02 | global_load_balancing_loss: 1.002959E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11177.660 | params rms: 0.060523 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:38:34.418671 | finish at 2026-10-17 17:59:45 + [2026-10-14 20:24:50] iteration 1249/ 2384 | consumed samples: 39968 | elapsed time per iteration (ms): 219737.9 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 76351.0 | valid tokens/s: 76351.0 | physical tokens/s: 76676.1 | loss tokens/s: 55845.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.318657E-01 | lm-global-supervised-tokens: 1.227144E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.809693E-02 | lm-token-buffer-mib: 2.326753E+02 | global_load_balancing_loss: 1.001042E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11177.625 | params rms: 0.060523 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:16:42.543041 | finish at 2026-10-17 17:41:33 + [2026-10-14 20:28:54] iteration 1250/ 2384 | consumed samples: 40000 | elapsed time per iteration (ms): 243568.1 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.10% | FLOPs estimator: workload-aware | logical tokens/s: 68880.9 | valid tokens/s: 68880.9 | physical tokens/s: 69826.2 | loss tokens/s: 47483.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.887275E-01 | lm-global-supervised-tokens: 1.156534E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.429311E-02 | lm-token-buffer-mib: 2.358051E+02 | global_load_balancing_loss: 1.000083E+00 | loss scale: 1.0 | grad norm: 0.341 | params norm: 11177.590 | params rms: 0.060523 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:43:26.194821 | finish at 2026-10-18 01:12:20 + [2026-10-14 20:32:36] iteration 1251/ 2384 | consumed samples: 40032 | elapsed time per iteration (ms): 222069.5 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75549.3 | valid tokens/s: 75549.3 | physical tokens/s: 76304.5 | loss tokens/s: 59326.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.810990E-01 | lm-global-supervised-tokens: 1.317452E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.858446E-02 | lm-token-buffer-mib: 2.347111E+02 | global_load_balancing_loss: 9.988320E-01 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11177.554 | params rms: 0.060523 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:53:24.799652 | finish at 2026-10-17 18:26:00 + [2026-10-14 20:36:20] iteration 1252/ 2384 | consumed samples: 40064 | elapsed time per iteration (ms): 223855.4 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 74946.5 | valid tokens/s: 74946.5 | physical tokens/s: 75541.9 | loss tokens/s: 58203.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.471251E-01 | lm-global-supervised-tokens: 1.302913E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.556551E-02 | lm-token-buffer-mib: 2.350184E+02 | global_load_balancing_loss: 9.981052E-01 | loss scale: 1.0 | grad norm: 0.214 | params norm: 11177.520 | params rms: 0.060523 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:23:24.275112 | finish at 2026-10-17 18:59:44 + [2026-10-14 20:40:00] iteration 1253/ 2384 | consumed samples: 40096 | elapsed time per iteration (ms): 220437.5 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 76108.6 | valid tokens/s: 76108.6 | physical tokens/s: 76531.2 | loss tokens/s: 54439.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.878500E-01 | lm-global-supervised-tokens: 1.200045E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.313286E-02 | lm-token-buffer-mib: 2.334520E+02 | global_load_balancing_loss: 9.987649E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11177.488 | params rms: 0.060522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:15:14.800905 | finish at 2026-10-17 17:55:15 + [2026-10-14 20:43:44] iteration 1254/ 2384 | consumed samples: 40128 | elapsed time per iteration (ms): 224216.0 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 74826.0 | valid tokens/s: 74826.0 | physical tokens/s: 75706.1 | loss tokens/s: 56638.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.902056E-01 | lm-global-supervised-tokens: 1.269930E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.255784E-02 | lm-token-buffer-mib: 2.354034E+02 | global_load_balancing_loss: 9.998629E-01 | loss scale: 1.0 | grad norm: 0.229 | params norm: 11177.454 | params rms: 0.060522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:22:44.108918 | finish at 2026-10-17 19:06:28 + [2026-10-14 20:47:29] iteration 1255/ 2384 | consumed samples: 40160 | elapsed time per iteration (ms): 224519.7 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.03% | FLOPs estimator: workload-aware | logical tokens/s: 74724.9 | valid tokens/s: 74724.9 | physical tokens/s: 75240.6 | loss tokens/s: 56192.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.109002E-01 | lm-global-supervised-tokens: 1.261624E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.332736E-02 | lm-token-buffer-mib: 2.334363E+02 | global_load_balancing_loss: 1.000342E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11177.420 | params rms: 0.060522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:24:42.744856 | finish at 2026-10-17 19:12:11 + [2026-10-14 20:51:13] iteration 1256/ 2384 | consumed samples: 40192 | elapsed time per iteration (ms): 224676.4 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.06% | FLOPs estimator: workload-aware | logical tokens/s: 74672.7 | valid tokens/s: 74672.7 | physical tokens/s: 75244.5 | loss tokens/s: 59505.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.867020E-01 | lm-global-supervised-tokens: 1.336939E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.612704E-02 | lm-token-buffer-mib: 2.339422E+02 | global_load_balancing_loss: 1.001910E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11177.386 | params rms: 0.060522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:23:54.981560 | finish at 2026-10-17 19:15:08 + [2026-10-14 20:54:55] iteration 1257/ 2384 | consumed samples: 40224 | elapsed time per iteration (ms): 221870.9 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 75616.9 | valid tokens/s: 75616.9 | physical tokens/s: 76300.7 | loss tokens/s: 57198.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.144432E-01 | lm-global-supervised-tokens: 1.269064E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.750879E-02 | lm-token-buffer-mib: 2.340031E+02 | global_load_balancing_loss: 1.001089E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11177.350 | params rms: 0.060522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:27:28.494801 | finish at 2026-10-17 18:22:24 + [2026-10-14 20:58:36] iteration 1258/ 2384 | consumed samples: 40256 | elapsed time per iteration (ms): 220979.0 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 75922.1 | valid tokens/s: 75922.1 | physical tokens/s: 76438.3 | loss tokens/s: 55943.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.437981E-01 | lm-global-supervised-tokens: 1.236233E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.354886E-02 | lm-token-buffer-mib: 2.324836E+02 | global_load_balancing_loss: 1.002437E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11177.316 | params rms: 0.060522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:07:02.347660 | finish at 2026-10-17 18:05:39 + [2026-10-14 21:02:21] iteration 1259/ 2384 | consumed samples: 40288 | elapsed time per iteration (ms): 224659.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 74678.5 | valid tokens/s: 74678.5 | physical tokens/s: 75280.5 | loss tokens/s: 57292.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.851910E-01 | lm-global-supervised-tokens: 1.287126E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.417383E-02 | lm-token-buffer-mib: 2.349109E+02 | global_load_balancing_loss: 1.001100E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11177.285 | params rms: 0.060521 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:12:21.321266 | finish at 2026-10-17 19:14:42 + [2026-10-14 21:06:27] iteration 1260/ 2384 | consumed samples: 40320 | elapsed time per iteration (ms): 246110.4 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 68169.2 | valid tokens/s: 68169.2 | physical tokens/s: 69045.3 | loss tokens/s: 51833.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.322473E-01 | lm-global-supervised-tokens: 1.275684E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.705777E-02 | lm-token-buffer-mib: 2.360014E+02 | global_load_balancing_loss: 1.001807E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11177.251 | params rms: 0.060521 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:50:28.129754 | finish at 2026-10-18 01:56:55 + [2026-10-14 21:10:14] iteration 1261/ 2384 | consumed samples: 40352 | elapsed time per iteration (ms): 226604.7 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 74037.3 | valid tokens/s: 74037.3 | physical tokens/s: 75357.2 | loss tokens/s: 51986.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.267999E-01 | lm-global-supervised-tokens: 1.178048E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.718702E-02 | lm-token-buffer-mib: 2.383908E+02 | global_load_balancing_loss: 9.988410E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11177.218 | params rms: 0.060521 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:41:17.070167 | finish at 2026-10-17 19:51:31 + [2026-10-14 21:13:58] iteration 1262/ 2384 | consumed samples: 40384 | elapsed time per iteration (ms): 224066.8 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 74875.9 | valid tokens/s: 74875.9 | physical tokens/s: 75681.4 | loss tokens/s: 56206.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.970893E-01 | lm-global-supervised-tokens: 1.259399E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.031410E-02 | lm-token-buffer-mib: 2.356028E+02 | global_load_balancing_loss: 9.985445E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11177.184 | params rms: 0.060521 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:50:02.957489 | finish at 2026-10-17 19:04:01 + [2026-10-14 21:17:41] iteration 1263/ 2384 | consumed samples: 40416 | elapsed time per iteration (ms): 223552.5 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.32% | FLOPs estimator: workload-aware | logical tokens/s: 75048.1 | valid tokens/s: 75048.1 | physical tokens/s: 75877.3 | loss tokens/s: 58109.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.482622E-01 | lm-global-supervised-tokens: 1.299046E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.567968E-02 | lm-token-buffer-mib: 2.366907E+02 | global_load_balancing_loss: 9.984522E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11177.151 | params rms: 0.060521 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:36:42.359192 | finish at 2026-10-17 18:54:24 + [2026-10-14 21:21:26] iteration 1264/ 2384 | consumed samples: 40448 | elapsed time per iteration (ms): 224270.7 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.05% | FLOPs estimator: workload-aware | logical tokens/s: 74807.8 | valid tokens/s: 74807.8 | physical tokens/s: 75841.5 | loss tokens/s: 54568.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.289764E-01 | lm-global-supervised-tokens: 1.223801E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.364700E-02 | lm-token-buffer-mib: 2.339952E+02 | global_load_balancing_loss: 1.002698E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11177.120 | params rms: 0.060520 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:46:23.190117 | finish at 2026-10-17 19:07:49 + [2026-10-14 21:25:09] iteration 1265/ 2384 | consumed samples: 40480 | elapsed time per iteration (ms): 223467.9 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.67% | FLOPs estimator: workload-aware | logical tokens/s: 75076.5 | valid tokens/s: 75076.5 | physical tokens/s: 75501.4 | loss tokens/s: 55685.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.237546E-01 | lm-global-supervised-tokens: 1.244399E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.243407E-02 | lm-token-buffer-mib: 2.329034E+02 | global_load_balancing_loss: 1.000365E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11177.088 | params rms: 0.060520 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:27:40.534521 | finish at 2026-10-17 18:52:50 + [2026-10-14 21:28:49] iteration 1266/ 2384 | consumed samples: 40512 | elapsed time per iteration (ms): 220200.7 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 76190.5 | valid tokens/s: 76190.5 | physical tokens/s: 76739.6 | loss tokens/s: 56120.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.862049E-01 | lm-global-supervised-tokens: 1.235786E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.173953E-02 | lm-token-buffer-mib: 2.333843E+02 | global_load_balancing_loss: 1.001253E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11177.050 | params rms: 0.060520 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:23:04.343467 | finish at 2026-10-17 17:51:54 + [2026-10-14 21:32:34] iteration 1267/ 2384 | consumed samples: 40544 | elapsed time per iteration (ms): 224623.8 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 74690.2 | valid tokens/s: 74690.2 | physical tokens/s: 75228.2 | loss tokens/s: 52957.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.517867E-01 | lm-global-supervised-tokens: 1.189560E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.236799E-02 | lm-token-buffer-mib: 2.338669E+02 | global_load_balancing_loss: 9.999951E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11177.007 | params rms: 0.060520 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:41:44.838439 | finish at 2026-10-17 19:14:19 + [2026-10-14 21:36:16] iteration 1268/ 2384 | consumed samples: 40576 | elapsed time per iteration (ms): 222008.6 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.07% | FLOPs estimator: workload-aware | logical tokens/s: 75569.9 | valid tokens/s: 75569.9 | physical tokens/s: 75831.6 | loss tokens/s: 58193.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.701863E-01 | lm-global-supervised-tokens: 1.291938E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.000175E-02 | lm-token-buffer-mib: 2.324527E+02 | global_load_balancing_loss: 1.002496E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11176.964 | params rms: 0.060520 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:49:21.608771 | finish at 2026-10-17 18:25:37 + [2026-10-14 21:40:05] iteration 1269/ 2384 | consumed samples: 40608 | elapsed time per iteration (ms): 228954.7 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 73277.3 | valid tokens/s: 73277.3 | physical tokens/s: 73569.0 | loss tokens/s: 56908.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.412135E-01 | lm-global-supervised-tokens: 1.302936E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.327611E-02 | lm-token-buffer-mib: 2.321625E+02 | global_load_balancing_loss: 1.001800E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11176.924 | params rms: 0.060519 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:54:44.513620 | finish at 2026-10-17 20:34:49 + [2026-10-14 21:44:09] iteration 1270/ 2384 | consumed samples: 40640 | elapsed time per iteration (ms): 244646.2 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 68577.3 | valid tokens/s: 68577.3 | physical tokens/s: 69429.2 | loss tokens/s: 50448.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.955661E-01 | lm-global-supervised-tokens: 1.234205E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.777480E-02 | lm-token-buffer-mib: 2.335748E+02 | global_load_balancing_loss: 9.989657E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11176.886 | params rms: 0.060519 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:42:15.873109 | finish at 2026-10-18 01:26:25 + [2026-10-14 21:47:54] iteration 1271/ 2384 | consumed samples: 40672 | elapsed time per iteration (ms): 224924.5 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 74590.2 | valid tokens/s: 74590.2 | physical tokens/s: 75242.6 | loss tokens/s: 55585.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.573059E-01 | lm-global-supervised-tokens: 1.250245E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.620357E-02 | lm-token-buffer-mib: 2.350149E+02 | global_load_balancing_loss: 9.998927E-01 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11176.847 | params rms: 0.060519 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:32:20.987062 | finish at 2026-10-17 19:20:15 + [2026-10-14 21:51:35] iteration 1272/ 2384 | consumed samples: 40704 | elapsed time per iteration (ms): 220256.0 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 76171.4 | valid tokens/s: 76171.4 | physical tokens/s: 76643.6 | loss tokens/s: 58476.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.949858E-01 | lm-global-supervised-tokens: 1.287977E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.610703E-02 | lm-token-buffer-mib: 2.331684E+02 | global_load_balancing_loss: 1.000364E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11176.813 | params rms: 0.060519 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:02:04.659321 | finish at 2026-10-17 17:53:39 + [2026-10-14 21:55:15] iteration 1273/ 2384 | consumed samples: 40736 | elapsed time per iteration (ms): 220771.0 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 75993.5 | valid tokens/s: 75993.5 | physical tokens/s: 76568.6 | loss tokens/s: 51404.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.598758E-01 | lm-global-supervised-tokens: 1.134858E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.325047E-02 | lm-token-buffer-mib: 2.323021E+02 | global_load_balancing_loss: 9.996255E-01 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11176.780 | params rms: 0.060519 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:07:56.571627 | finish at 2026-10-17 18:03:12 + [2026-10-14 21:58:56] iteration 1274/ 2384 | consumed samples: 40768 | elapsed time per iteration (ms): 220442.0 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 76107.1 | valid tokens/s: 76107.1 | physical tokens/s: 76728.8 | loss tokens/s: 57706.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.324545E-01 | lm-global-supervised-tokens: 1.272088E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.696677E-02 | lm-token-buffer-mib: 2.343628E+02 | global_load_balancing_loss: 1.002103E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11176.746 | params rms: 0.060518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:58:10.568826 | finish at 2026-10-17 17:57:06 + [2026-10-14 22:02:40] iteration 1275/ 2384 | consumed samples: 40800 | elapsed time per iteration (ms): 224598.5 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 74698.5 | valid tokens/s: 74698.5 | physical tokens/s: 75860.7 | loss tokens/s: 57933.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.766239E-01 | lm-global-supervised-tokens: 1.301170E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.666597E-02 | lm-token-buffer-mib: 2.359669E+02 | global_load_balancing_loss: 1.001949E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11176.713 | params rms: 0.060518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:11:19.700556 | finish at 2026-10-17 19:14:00 + [2026-10-14 22:06:27] iteration 1276/ 2384 | consumed samples: 40832 | elapsed time per iteration (ms): 226477.2 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 74079.0 | valid tokens/s: 74079.0 | physical tokens/s: 74562.8 | loss tokens/s: 58804.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.737236E-01 | lm-global-supervised-tokens: 1.331795E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.256543E-02 | lm-token-buffer-mib: 2.344532E+02 | global_load_balancing_loss: 1.001161E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11176.683 | params rms: 0.060518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:42:16.755862 | finish at 2026-10-17 19:48:44 + [2026-10-14 22:10:09] iteration 1277/ 2384 | consumed samples: 40864 | elapsed time per iteration (ms): 222004.9 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 75571.3 | valid tokens/s: 75571.3 | physical tokens/s: 76058.3 | loss tokens/s: 55390.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.446925E-01 | lm-global-supervised-tokens: 1.229702E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.486533E-02 | lm-token-buffer-mib: 2.339495E+02 | global_load_balancing_loss: 1.000619E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11176.650 | params rms: 0.060518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:15:59.393397 | finish at 2026-10-17 18:26:08 + [2026-10-14 22:13:50] iteration 1278/ 2384 | consumed samples: 40896 | elapsed time per iteration (ms): 221480.4 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.29% | FLOPs estimator: workload-aware | logical tokens/s: 75750.3 | valid tokens/s: 75750.3 | physical tokens/s: 77282.2 | loss tokens/s: 56928.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.965771E-01 | lm-global-supervised-tokens: 1.260856E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.999063E-02 | lm-token-buffer-mib: 2.362835E+02 | global_load_balancing_loss: 1.000029E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11176.620 | params rms: 0.060518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:02:37.273184 | finish at 2026-10-17 18:16:28 + [2026-10-14 22:17:33] iteration 1279/ 2384 | consumed samples: 40928 | elapsed time per iteration (ms): 222611.0 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 75365.4 | valid tokens/s: 75365.4 | physical tokens/s: 76296.8 | loss tokens/s: 57355.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.972287E-01 | lm-global-supervised-tokens: 1.276791E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.412670E-02 | lm-token-buffer-mib: 2.356975E+02 | global_load_balancing_loss: 1.001730E+00 | loss scale: 1.0 | grad norm: 0.508 | params norm: 11176.589 | params rms: 0.060518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:19:45.141631 | finish at 2026-10-17 18:37:18 + [2026-10-14 22:21:35] iteration 1280/ 2384 | consumed samples: 40960 | elapsed time per iteration (ms): 241762.2 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.33% | FLOPs estimator: workload-aware | logical tokens/s: 69395.3 | valid tokens/s: 69395.3 | physical tokens/s: 69890.0 | loss tokens/s: 53387.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.222549E-01 | lm-global-supervised-tokens: 1.290710E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.604214E-02 | lm-token-buffer-mib: 2.338498E+02 | global_load_balancing_loss: 1.000647E+00 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11176.557 | params rms: 0.060517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:08:25.514729 | finish at 2026-10-18 00:30:00 +times across ranks (ms): + save-checkpoint: + rank 0: 24842.32 + rank 1: 24842.32 + rank 2: 24842.31 + rank 3: 24842.32 + rank 4: 24842.30 + rank 5: 24842.31 + rank 6: 24842.35 + rank 7: 24842.29 + rank 8: 24842.96 + rank 9: 24842.97 + rank 10: 24842.96 + rank 11: 24842.96 + rank 12: 24842.94 + rank 13: 24842.95 + rank 14: 24842.95 + rank 15: 24842.97 + rank 16: 24842.99 + rank 17: 24842.99 + rank 18: 24843.00 + rank 19: 24842.98 + rank 20: 24842.96 + rank 21: 24843.02 + rank 22: 24842.87 + rank 23: 24842.97 + rank 24: 24842.69 + rank 25: 24842.67 + rank 26: 24842.74 + rank 27: 24843.45 + rank 28: 24842.67 + rank 29: 24842.69 + rank 30: 24842.76 + rank 31: 24842.56 + rank 32: 24842.44 + rank 33: 24842.17 + rank 34: 24842.06 + rank 35: 24842.47 + rank 36: 24842.18 + rank 37: 24842.20 + rank 38: 24842.15 + rank 39: 24842.19 + rank 40: 24842.33 + rank 41: 24842.32 + rank 42: 24842.31 + rank 43: 24842.32 + rank 44: 24842.30 + rank 45: 24842.30 + rank 46: 24842.34 + rank 47: 24842.30 + rank 48: 24841.77 + rank 49: 24841.81 + rank 50: 24841.83 + rank 51: 24841.79 + rank 52: 24841.79 + rank 53: 24841.81 + rank 54: 24841.76 + rank 55: 24841.79 + rank 56: 24842.76 + rank 57: 24842.79 + rank 58: 24842.79 + rank 59: 24842.81 + rank 60: 24842.79 + rank 61: 24842.75 + rank 62: 24842.76 + rank 63: 24842.75 + [2026-10-14 22:26:01] iteration 1281/ 2384 | consumed samples: 40992 | elapsed time per iteration (ms): 241243.9 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.38% | FLOPs estimator: workload-aware | logical tokens/s: 69544.5 | valid tokens/s: 69544.5 | physical tokens/s: 70731.0 | loss tokens/s: 52902.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.015213E-01 | lm-global-supervised-tokens: 1.276228E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.094176E+00 | lm-token-buffer-mib: 2.330952E+02 | global_load_balancing_loss: 9.995441E-01 | loss scale: 1.0 | grad norm: 0.337 | params norm: 11176.523 | params rms: 0.060517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:54:52.036494 | finish at 2026-10-18 00:20:53 + [2026-10-14 22:29:54] iteration 1282/ 2384 | consumed samples: 41024 | elapsed time per iteration (ms): 232923.6 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.11% | FLOPs estimator: workload-aware | logical tokens/s: 72028.8 | valid tokens/s: 72028.8 | physical tokens/s: 72718.0 | loss tokens/s: 53630.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.776293E-01 | lm-global-supervised-tokens: 1.249170E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.890841E-02 | lm-token-buffer-mib: 2.356841E+02 | global_load_balancing_loss: 1.001030E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11176.493 | params rms: 0.060517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:18:01.826859 | finish at 2026-10-17 21:47:56 + [2026-10-14 22:33:36] iteration 1283/ 2384 | consumed samples: 41056 | elapsed time per iteration (ms): 222201.6 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 75504.3 | valid tokens/s: 75504.3 | physical tokens/s: 76252.8 | loss tokens/s: 58939.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.730556E-01 | lm-global-supervised-tokens: 1.309649E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.594632E-02 | lm-token-buffer-mib: 2.345481E+02 | global_load_balancing_loss: 9.990497E-01 | loss scale: 1.0 | grad norm: 0.422 | params norm: 11176.462 | params rms: 0.060517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:57:23.988457 | finish at 2026-10-17 18:31:00 + [2026-10-14 22:37:18] iteration 1284/ 2384 | consumed samples: 41088 | elapsed time per iteration (ms): 221863.9 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75619.4 | valid tokens/s: 75619.4 | physical tokens/s: 76040.3 | loss tokens/s: 56562.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.032342E-01 | lm-global-supervised-tokens: 1.254924E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.546066E-02 | lm-token-buffer-mib: 2.342097E+02 | global_load_balancing_loss: 9.990722E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11176.430 | params rms: 0.060517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:47:30.241423 | finish at 2026-10-17 18:24:48 + [2026-10-14 22:41:00] iteration 1285/ 2384 | consumed samples: 41120 | elapsed time per iteration (ms): 221328.9 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 75802.1 | valid tokens/s: 75802.1 | physical tokens/s: 75955.2 | loss tokens/s: 54321.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.528326E-01 | lm-global-supervised-tokens: 1.202283E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.129630E-02 | lm-token-buffer-mib: 2.325312E+02 | global_load_balancing_loss: 1.000122E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11176.392 | params rms: 0.060517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:34:00.449156 | finish at 2026-10-17 18:15:00 + [2026-10-14 22:44:41] iteration 1286/ 2384 | consumed samples: 41152 | elapsed time per iteration (ms): 221597.6 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.36% | FLOPs estimator: workload-aware | logical tokens/s: 75710.1 | valid tokens/s: 75710.1 | physical tokens/s: 76212.2 | loss tokens/s: 58126.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.856153E-01 | lm-global-supervised-tokens: 1.288065E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.711792E-02 | lm-token-buffer-mib: 2.338337E+02 | global_load_balancing_loss: 1.000085E+00 | loss scale: 1.0 | grad norm: 0.193 | params norm: 11176.356 | params rms: 0.060516 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:35:14.171326 | finish at 2026-10-17 18:19:55 + [2026-10-14 22:48:25] iteration 1287/ 2384 | consumed samples: 41184 | elapsed time per iteration (ms): 223585.1 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 75037.1 | valid tokens/s: 75037.1 | physical tokens/s: 76157.3 | loss tokens/s: 55588.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.296380E-01 | lm-global-supervised-tokens: 1.242868E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.361084E-02 | lm-token-buffer-mib: 2.362611E+02 | global_load_balancing_loss: 1.002168E+00 | loss scale: 1.0 | grad norm: 0.268 | params norm: 11176.318 | params rms: 0.060516 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:07:52.891246 | finish at 2026-10-17 18:56:18 + [2026-10-14 22:52:06] iteration 1288/ 2384 | consumed samples: 41216 | elapsed time per iteration (ms): 220980.7 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 75921.5 | valid tokens/s: 75921.5 | physical tokens/s: 76388.8 | loss tokens/s: 57779.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.989081E-01 | lm-global-supervised-tokens: 1.276812E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.819980E-02 | lm-token-buffer-mib: 2.325237E+02 | global_load_balancing_loss: 1.002552E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11176.280 | params rms: 0.060516 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:16:34.842253 | finish at 2026-10-17 18:08:41 + [2026-10-14 22:55:50] iteration 1289/ 2384 | consumed samples: 41248 | elapsed time per iteration (ms): 224108.1 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.22% | FLOPs estimator: workload-aware | logical tokens/s: 74861.9 | valid tokens/s: 74861.9 | physical tokens/s: 75473.0 | loss tokens/s: 56569.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.280863E-01 | lm-global-supervised-tokens: 1.267778E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.053118E-02 | lm-token-buffer-mib: 2.342397E+02 | global_load_balancing_loss: 1.000130E+00 | loss scale: 1.0 | grad norm: 0.204 | params norm: 11176.241 | params rms: 0.060516 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:09:58.401543 | finish at 2026-10-17 19:05:48 + [2026-10-14 22:59:54] iteration 1290/ 2384 | consumed samples: 41280 | elapsed time per iteration (ms): 243722.1 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.95% | FLOPs estimator: workload-aware | logical tokens/s: 68837.4 | valid tokens/s: 68837.4 | physical tokens/s: 69431.5 | loss tokens/s: 52407.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.954682E-01 | lm-global-supervised-tokens: 1.277287E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.381347E-02 | lm-token-buffer-mib: 2.337972E+02 | global_load_balancing_loss: 1.001739E+00 | loss scale: 1.0 | grad norm: 0.238 | params norm: 11176.201 | params rms: 0.060515 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:03:51.970640 | finish at 2026-10-18 01:03:46 + [2026-10-14 23:03:37] iteration 1291/ 2384 | consumed samples: 41312 | elapsed time per iteration (ms): 223573.8 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.38% | FLOPs estimator: workload-aware | logical tokens/s: 75040.9 | valid tokens/s: 75040.9 | physical tokens/s: 75604.7 | loss tokens/s: 60033.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.488728E-01 | lm-global-supervised-tokens: 1.342187E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.027796E-01 | lm-token-buffer-mib: 2.344908E+02 | global_load_balancing_loss: 1.002146E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11176.160 | params rms: 0.060515 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:52:46.174179 | finish at 2026-10-17 18:56:23 + [2026-10-14 23:07:19] iteration 1292/ 2384 | consumed samples: 41344 | elapsed time per iteration (ms): 221519.8 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.91% | FLOPs estimator: workload-aware | logical tokens/s: 75736.7 | valid tokens/s: 75736.7 | physical tokens/s: 76035.3 | loss tokens/s: 58286.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.176927E-01 | lm-global-supervised-tokens: 1.291150E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.909728E-02 | lm-token-buffer-mib: 2.330896E+02 | global_load_balancing_loss: 9.993222E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11176.121 | params rms: 0.060515 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:11:39.569733 | finish at 2026-10-17 18:18:58 + [2026-10-14 23:11:04] iteration 1293/ 2384 | consumed samples: 41376 | elapsed time per iteration (ms): 225780.2 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 74307.7 | valid tokens/s: 74307.7 | physical tokens/s: 74800.7 | loss tokens/s: 54240.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.231477E-01 | lm-global-supervised-tokens: 1.224652E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.400769E-02 | lm-token-buffer-mib: 2.324176E+02 | global_load_balancing_loss: 9.983402E-01 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11176.080 | params rms: 0.060515 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:25:26.172454 | finish at 2026-10-17 19:36:31 + [2026-10-14 23:14:46] iteration 1294/ 2384 | consumed samples: 41408 | elapsed time per iteration (ms): 221618.2 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 75703.1 | valid tokens/s: 75703.1 | physical tokens/s: 76013.7 | loss tokens/s: 59518.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.733422E-01 | lm-global-supervised-tokens: 1.319038E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.927704E-02 | lm-token-buffer-mib: 2.331229E+02 | global_load_balancing_loss: 9.983067E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11176.042 | params rms: 0.060515 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:06:03.798048 | finish at 2026-10-17 18:20:50 + [2026-10-14 23:18:33] iteration 1295/ 2384 | consumed samples: 41440 | elapsed time per iteration (ms): 227229.3 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 73833.8 | valid tokens/s: 73833.8 | physical tokens/s: 74363.7 | loss tokens/s: 53670.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.111051E-01 | lm-global-supervised-tokens: 1.219552E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.575687E-02 | lm-token-buffer-mib: 2.341799E+02 | global_load_balancing_loss: 9.991791E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11176.005 | params rms: 0.060514 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:44:12.700974 | finish at 2026-10-17 20:02:46 + [2026-10-14 23:22:18] iteration 1296/ 2384 | consumed samples: 41472 | elapsed time per iteration (ms): 224773.5 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 74640.5 | valid tokens/s: 74640.5 | physical tokens/s: 75464.6 | loss tokens/s: 57707.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.149099E-01 | lm-global-supervised-tokens: 1.297102E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.622188E-02 | lm-token-buffer-mib: 2.357023E+02 | global_load_balancing_loss: 1.001001E+00 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11175.965 | params rms: 0.060514 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:55:53.522308 | finish at 2026-10-17 19:18:12 + [2026-10-14 23:26:00] iteration 1297/ 2384 | consumed samples: 41504 | elapsed time per iteration (ms): 222314.9 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 75465.8 | valid tokens/s: 75465.8 | physical tokens/s: 76306.6 | loss tokens/s: 54721.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.068601E-01 | lm-global-supervised-tokens: 1.216530E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 8.961512E-02 | lm-token-buffer-mib: 2.329825E+02 | global_load_balancing_loss: 1.001256E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11175.925 | params rms: 0.060514 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:07:36.286366 | finish at 2026-10-17 18:33:37 + [2026-10-14 23:29:39] iteration 1298/ 2384 | consumed samples: 41536 | elapsed time per iteration (ms): 218893.0 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 76645.5 | valid tokens/s: 76645.5 | physical tokens/s: 76957.4 | loss tokens/s: 57337.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.400968E-01 | lm-global-supervised-tokens: 1.255066E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.947629E-02 | lm-token-buffer-mib: 2.325967E+02 | global_load_balancing_loss: 1.001743E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11175.884 | params rms: 0.060514 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:01:57.770950 | finish at 2026-10-17 17:31:37 + [2026-10-14 23:33:22] iteration 1299/ 2384 | consumed samples: 41568 | elapsed time per iteration (ms): 222475.4 | throughput per GPU (TFLOP/s/GPU): 27.0 | MFU 8.66% | FLOPs estimator: workload-aware | logical tokens/s: 75411.3 | valid tokens/s: 75411.3 | physical tokens/s: 75822.9 | loss tokens/s: 53303.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.860696E-01 | lm-global-supervised-tokens: 1.185860E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.252704E-02 | lm-token-buffer-mib: 2.335641E+02 | global_load_balancing_loss: 1.000580E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11175.844 | params rms: 0.060514 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:03:05.766027 | finish at 2026-10-17 18:36:28 + [2026-10-14 23:37:22] iteration 1300/ 2384 | consumed samples: 41600 | elapsed time per iteration (ms): 240667.8 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.21% | FLOPs estimator: workload-aware | logical tokens/s: 69711.0 | valid tokens/s: 69711.0 | physical tokens/s: 70294.5 | loss tokens/s: 52895.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.815898E-01 | lm-global-supervised-tokens: 1.273030E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.844087E-02 | lm-token-buffer-mib: 2.341000E+02 | global_load_balancing_loss: 1.001469E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11175.798 | params rms: 0.060513 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:28:03.887650 | finish at 2026-10-18 00:05:26 + [2026-10-14 23:41:04] iteration 1301/ 2384 | consumed samples: 41632 | elapsed time per iteration (ms): 221914.3 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.83% | FLOPs estimator: workload-aware | logical tokens/s: 75602.0 | valid tokens/s: 75602.0 | physical tokens/s: 75822.6 | loss tokens/s: 50419.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.254995E-01 | lm-global-supervised-tokens: 1.118885E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.458145E-02 | lm-token-buffer-mib: 2.327769E+02 | global_load_balancing_loss: 1.001054E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11175.756 | params rms: 0.060513 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:45:33.204162 | finish at 2026-10-17 18:26:38 + [2026-10-14 23:44:47] iteration 1302/ 2384 | consumed samples: 41664 | elapsed time per iteration (ms): 223026.5 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.91% | FLOPs estimator: workload-aware | logical tokens/s: 75225.1 | valid tokens/s: 75225.1 | physical tokens/s: 75909.3 | loss tokens/s: 54900.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.559610E-01 | lm-global-supervised-tokens: 1.224432E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.284562E-02 | lm-token-buffer-mib: 2.350737E+02 | global_load_balancing_loss: 1.000501E+00 | loss scale: 1.0 | grad norm: 0.359 | params norm: 11175.718 | params rms: 0.060513 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:01:54.674533 | finish at 2026-10-17 18:46:42 + [2026-10-14 23:48:30] iteration 1303/ 2384 | consumed samples: 41696 | elapsed time per iteration (ms): 222335.2 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 75459.0 | valid tokens/s: 75459.0 | physical tokens/s: 75998.2 | loss tokens/s: 55037.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.524512E-01 | lm-global-supervised-tokens: 1.223685E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.872783E-02 | lm-token-buffer-mib: 2.327791E+02 | global_load_balancing_loss: 1.000704E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11175.679 | params rms: 0.060513 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:45:44.391483 | finish at 2026-10-17 18:34:14 + [2026-10-14 23:52:12] iteration 1304/ 2384 | consumed samples: 41728 | elapsed time per iteration (ms): 221934.3 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 75595.2 | valid tokens/s: 75595.2 | physical tokens/s: 76004.3 | loss tokens/s: 57159.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.759724E-01 | lm-global-supervised-tokens: 1.268568E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.341100E-02 | lm-token-buffer-mib: 2.329281E+02 | global_load_balancing_loss: 9.995574E-01 | loss scale: 1.0 | grad norm: 0.414 | params norm: 11175.638 | params rms: 0.060512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:34:49.012012 | finish at 2026-10-17 18:27:01 + [2026-10-14 23:55:58] iteration 1305/ 2384 | consumed samples: 41760 | elapsed time per iteration (ms): 226314.0 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 74132.4 | valid tokens/s: 74132.4 | physical tokens/s: 74793.9 | loss tokens/s: 57065.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.421999E-01 | lm-global-supervised-tokens: 1.291464E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.346539E-02 | lm-token-buffer-mib: 2.339203E+02 | global_load_balancing_loss: 1.000914E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11175.599 | params rms: 0.060512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:49:52.784531 | finish at 2026-10-17 19:45:51 + [2026-10-14 23:59:40] iteration 1306/ 2384 | consumed samples: 41792 | elapsed time per iteration (ms): 221774.0 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 75649.8 | valid tokens/s: 75649.8 | physical tokens/s: 76231.0 | loss tokens/s: 57369.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.854420E-01 | lm-global-supervised-tokens: 1.272299E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.569380E-02 | lm-token-buffer-mib: 2.346671E+02 | global_load_balancing_loss: 9.993302E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11175.564 | params rms: 0.060512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:24:32.355218 | finish at 2026-10-17 18:24:12 + [2026-10-15 00:03:26] iteration 1307/ 2384 | consumed samples: 41824 | elapsed time per iteration (ms): 226046.2 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 74220.2 | valid tokens/s: 74220.2 | physical tokens/s: 75748.9 | loss tokens/s: 58161.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.317218E-01 | lm-global-supervised-tokens: 1.314708E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.504669E-02 | lm-token-buffer-mib: 2.354470E+02 | global_load_balancing_loss: 9.993517E-01 | loss scale: 1.0 | grad norm: 0.325 | params norm: 11175.532 | params rms: 0.060512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:37:31.739465 | finish at 2026-10-17 19:40:58 + [2026-10-15 00:07:12] iteration 1308/ 2384 | consumed samples: 41856 | elapsed time per iteration (ms): 226264.1 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 74148.5 | valid tokens/s: 74148.5 | physical tokens/s: 74573.9 | loss tokens/s: 54324.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.769445E-01 | lm-global-supervised-tokens: 1.229177E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.415602E-02 | lm-token-buffer-mib: 2.333476E+02 | global_load_balancing_loss: 9.999765E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11175.501 | params rms: 0.060512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:37:40.211187 | finish at 2026-10-17 19:44:52 + [2026-10-15 00:10:54] iteration 1309/ 2384 | consumed samples: 41888 | elapsed time per iteration (ms): 221738.0 | throughput per GPU (TFLOP/s/GPU): 26.0 | MFU 8.32% | FLOPs estimator: workload-aware | logical tokens/s: 75662.1 | valid tokens/s: 75662.1 | physical tokens/s: 76265.6 | loss tokens/s: 59430.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.188936E-01 | lm-global-supervised-tokens: 1.317805E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.961091E-02 | lm-token-buffer-mib: 2.342985E+02 | global_load_balancing_loss: 9.998565E-01 | loss scale: 1.0 | grad norm: 0.316 | params norm: 11175.468 | params rms: 0.060512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:12:48.302494 | finish at 2026-10-17 18:23:42 + [2026-10-15 00:15:00] iteration 1310/ 2384 | consumed samples: 41920 | elapsed time per iteration (ms): 246027.1 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 68192.4 | valid tokens/s: 68192.4 | physical tokens/s: 68910.5 | loss tokens/s: 52389.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.495626E-01 | lm-global-supervised-tokens: 1.288932E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.130080E-02 | lm-token-buffer-mib: 2.342597E+02 | global_load_balancing_loss: 1.001694E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11175.437 | params rms: 0.060511 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:23:53.071436 | finish at 2026-10-18 01:38:53 + [2026-10-15 00:18:42] iteration 1311/ 2384 | consumed samples: 41952 | elapsed time per iteration (ms): 221719.9 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 75668.3 | valid tokens/s: 75668.3 | physical tokens/s: 76579.8 | loss tokens/s: 61635.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.863480E-01 | lm-global-supervised-tokens: 1.366588E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.401114E-02 | lm-token-buffer-mib: 2.343807E+02 | global_load_balancing_loss: 1.003242E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11175.406 | params rms: 0.060511 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:05:05.472283 | finish at 2026-10-17 18:23:47 + [2026-10-15 00:22:23] iteration 1312/ 2384 | consumed samples: 41984 | elapsed time per iteration (ms): 221588.1 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 75713.5 | valid tokens/s: 75713.5 | physical tokens/s: 76468.2 | loss tokens/s: 57554.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.262135E-01 | lm-global-supervised-tokens: 1.275339E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.374623E-02 | lm-token-buffer-mib: 2.334535E+02 | global_load_balancing_loss: 1.001630E+00 | loss scale: 1.0 | grad norm: 0.284 | params norm: 11175.374 | params rms: 0.060511 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:59:02.483025 | finish at 2026-10-17 18:21:26 + [2026-10-15 00:26:09] iteration 1313/ 2384 | consumed samples: 42016 | elapsed time per iteration (ms): 225652.7 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.44% | FLOPs estimator: workload-aware | logical tokens/s: 74349.4 | valid tokens/s: 74349.4 | physical tokens/s: 75500.9 | loss tokens/s: 55272.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.122802E-01 | lm-global-supervised-tokens: 1.247230E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.148648E-02 | lm-token-buffer-mib: 2.362383E+02 | global_load_balancing_loss: 9.998723E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11175.341 | params rms: 0.060511 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:07:54.065135 | finish at 2026-10-17 19:34:03 + [2026-10-15 00:29:49] iteration 1314/ 2384 | consumed samples: 42048 | elapsed time per iteration (ms): 220245.7 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 76174.9 | valid tokens/s: 76174.9 | physical tokens/s: 76486.5 | loss tokens/s: 53445.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.507172E-01 | lm-global-supervised-tokens: 1.177115E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.631779E-02 | lm-token-buffer-mib: 2.333920E+02 | global_load_balancing_loss: 9.984787E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11175.305 | params rms: 0.060511 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:27:42.951682 | finish at 2026-10-17 17:57:32 + [2026-10-15 00:33:32] iteration 1315/ 2384 | consumed samples: 42080 | elapsed time per iteration (ms): 222803.6 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 75300.4 | valid tokens/s: 75300.4 | physical tokens/s: 75918.6 | loss tokens/s: 58043.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.264874E-01 | lm-global-supervised-tokens: 1.293229E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.663664E-02 | lm-token-buffer-mib: 2.332912E+02 | global_load_balancing_loss: 9.992537E-01 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11175.265 | params rms: 0.060510 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:09:37.037009 | finish at 2026-10-17 18:43:09 + [2026-10-15 00:37:12] iteration 1316/ 2384 | consumed samples: 42112 | elapsed time per iteration (ms): 220349.1 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 76139.2 | valid tokens/s: 76139.2 | physical tokens/s: 76540.7 | loss tokens/s: 55044.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.650304E-01 | lm-global-supervised-tokens: 1.212891E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.994440E-01 | lm-token-buffer-mib: 2.328863E+02 | global_load_balancing_loss: 1.000372E+00 | loss scale: 1.0 | grad norm: 0.212 | params norm: 11175.225 | params rms: 0.060510 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:22:12.820333 | finish at 2026-10-17 17:59:25 + [2026-10-15 00:40:54] iteration 1317/ 2384 | consumed samples: 42144 | elapsed time per iteration (ms): 221669.9 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.47% | FLOPs estimator: workload-aware | logical tokens/s: 75685.6 | valid tokens/s: 75685.6 | physical tokens/s: 76725.6 | loss tokens/s: 55426.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.664750E-01 | lm-global-supervised-tokens: 1.228628E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.485227E-02 | lm-token-buffer-mib: 2.337529E+02 | global_load_balancing_loss: 9.997655E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11175.191 | params rms: 0.060510 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:42:01.732612 | finish at 2026-10-17 18:22:56 + [2026-10-15 00:44:37] iteration 1318/ 2384 | consumed samples: 42176 | elapsed time per iteration (ms): 222784.9 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.75% | FLOPs estimator: workload-aware | logical tokens/s: 75306.5 | valid tokens/s: 75306.5 | physical tokens/s: 76107.4 | loss tokens/s: 61706.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.607452E-01 | lm-global-supervised-tokens: 1.374736E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.737499E-02 | lm-token-buffer-mib: 2.336595E+02 | global_load_balancing_loss: 1.000831E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11175.158 | params rms: 0.060510 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:58:08.728254 | finish at 2026-10-17 18:42:45 + [2026-10-15 00:48:17] iteration 1319/ 2384 | consumed samples: 42208 | elapsed time per iteration (ms): 220414.7 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 76116.5 | valid tokens/s: 76116.4 | physical tokens/s: 76844.5 | loss tokens/s: 60648.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.686843E-01 | lm-global-supervised-tokens: 1.336778E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.728852E-02 | lm-token-buffer-mib: 2.337238E+02 | global_load_balancing_loss: 1.001837E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11175.123 | params rms: 0.060510 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:12:21.703523 | finish at 2026-10-17 18:00:39 + [2026-10-15 00:52:24] iteration 1320/ 2384 | consumed samples: 42240 | elapsed time per iteration (ms): 247221.3 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 67862.9 | valid tokens/s: 67862.9 | physical tokens/s: 68697.0 | loss tokens/s: 52001.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.313896E-01 | lm-global-supervised-tokens: 1.285590E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.338692E-02 | lm-token-buffer-mib: 2.345769E+02 | global_load_balancing_loss: 1.001458E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11175.091 | params rms: 0.060509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:04:03.425028 | finish at 2026-10-18 01:56:28 +times across ranks (ms): + save-checkpoint: + rank 0: 24430.87 + rank 1: 24430.91 + rank 2: 24430.87 + rank 3: 24430.87 + rank 4: 24430.88 + rank 5: 24430.92 + rank 6: 24430.91 + rank 7: 24430.88 + rank 8: 24430.35 + rank 9: 24430.40 + rank 10: 24430.40 + rank 11: 24430.35 + rank 12: 24430.38 + rank 13: 24430.35 + rank 14: 24430.37 + rank 15: 24430.35 + rank 16: 24430.27 + rank 17: 24430.26 + rank 18: 24430.26 + rank 19: 24430.31 + rank 20: 24430.08 + rank 21: 24430.22 + rank 22: 24430.26 + rank 23: 24430.22 + rank 24: 24430.15 + rank 25: 24430.15 + rank 26: 24430.24 + rank 27: 24430.12 + rank 28: 24430.46 + rank 29: 24430.68 + rank 30: 24430.18 + rank 31: 24430.11 + rank 32: 24430.29 + rank 33: 24430.34 + rank 34: 24430.33 + rank 35: 24430.36 + rank 36: 24430.36 + rank 37: 24430.35 + rank 38: 24430.29 + rank 39: 24430.26 + rank 40: 24430.27 + rank 41: 24430.24 + rank 42: 24430.30 + rank 43: 24430.25 + rank 44: 24430.25 + rank 45: 24430.23 + rank 46: 24430.22 + rank 47: 24430.23 + rank 48: 24430.26 + rank 49: 24430.38 + rank 50: 24430.33 + rank 51: 24430.26 + rank 52: 24430.34 + rank 53: 24430.44 + rank 54: 24430.23 + rank 55: 24430.30 + rank 56: 24430.53 + rank 57: 24430.55 + rank 58: 24430.60 + rank 59: 24430.55 + rank 60: 24430.57 + rank 61: 24430.59 + rank 62: 24430.56 + rank 63: 24430.53 + [2026-10-15 00:56:30] iteration 1321/ 2384 | consumed samples: 42272 | elapsed time per iteration (ms): 221522.7 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75735.8 | valid tokens/s: 75735.8 | physical tokens/s: 75993.3 | loss tokens/s: 53968.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.377423E-01 | lm-global-supervised-tokens: 1.195515E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.467451E-02 | lm-token-buffer-mib: 2.331582E+02 | global_load_balancing_loss: 1.001418E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11175.061 | params rms: 0.060509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:24:38.682638 | finish at 2026-10-17 18:21:09 + [2026-10-15 01:00:18] iteration 1322/ 2384 | consumed samples: 42304 | elapsed time per iteration (ms): 227465.5 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 73757.1 | valid tokens/s: 73757.1 | physical tokens/s: 74682.0 | loss tokens/s: 55875.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.144879E-01 | lm-global-supervised-tokens: 1.270967E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.618419E-02 | lm-token-buffer-mib: 2.323827E+02 | global_load_balancing_loss: 9.991407E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11175.034 | params rms: 0.060509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:06:08.314028 | finish at 2026-10-17 20:06:26 + [2026-10-15 01:04:00] iteration 1323/ 2384 | consumed samples: 42336 | elapsed time per iteration (ms): 221894.6 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 75608.8 | valid tokens/s: 75608.8 | physical tokens/s: 76559.6 | loss tokens/s: 56397.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.432318E-01 | lm-global-supervised-tokens: 1.251422E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.721379E-02 | lm-token-buffer-mib: 2.353523E+02 | global_load_balancing_loss: 9.994416E-01 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11175.005 | params rms: 0.060509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:23:50.213766 | finish at 2026-10-17 18:27:50 + [2026-10-15 01:07:39] iteration 1324/ 2384 | consumed samples: 42368 | elapsed time per iteration (ms): 219658.3 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.21% | FLOPs estimator: workload-aware | logical tokens/s: 76378.5 | valid tokens/s: 76378.5 | physical tokens/s: 77024.4 | loss tokens/s: 60090.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.111025E-01 | lm-global-supervised-tokens: 1.319927E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.200153E-02 | lm-token-buffer-mib: 2.344075E+02 | global_load_balancing_loss: 9.981468E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11174.971 | params rms: 0.060509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:40:37.782755 | finish at 2026-10-17 17:48:17 + [2026-10-15 01:11:22] iteration 1325/ 2384 | consumed samples: 42400 | elapsed time per iteration (ms): 222044.2 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 75558.0 | valid tokens/s: 75558.0 | physical tokens/s: 76161.0 | loss tokens/s: 56032.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.371923E-01 | lm-global-supervised-tokens: 1.244170E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.732512E-02 | lm-token-buffer-mib: 2.340048E+02 | global_load_balancing_loss: 9.999579E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11174.935 | params rms: 0.060509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:19:04.760020 | finish at 2026-10-17 18:30:26 + [2026-10-15 01:15:04] iteration 1326/ 2384 | consumed samples: 42432 | elapsed time per iteration (ms): 222398.3 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.31% | FLOPs estimator: workload-aware | logical tokens/s: 75437.5 | valid tokens/s: 75437.5 | physical tokens/s: 76129.2 | loss tokens/s: 58528.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.766322E-01 | lm-global-supervised-tokens: 1.301661E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.358406E-02 | lm-token-buffer-mib: 2.345388E+02 | global_load_balancing_loss: 1.001106E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11174.900 | params rms: 0.060508 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:21:37.419995 | finish at 2026-10-17 18:36:41 + [2026-10-15 01:18:45] iteration 1327/ 2384 | consumed samples: 42464 | elapsed time per iteration (ms): 220564.2 | throughput per GPU (TFLOP/s/GPU): 17.2 | MFU 5.52% | FLOPs estimator: workload-aware | logical tokens/s: 76065.0 | valid tokens/s: 76065.0 | physical tokens/s: 76843.5 | loss tokens/s: 57562.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.433792E-01 | lm-global-supervised-tokens: 1.269631E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.570272E-02 | lm-token-buffer-mib: 2.340565E+02 | global_load_balancing_loss: 1.002286E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11174.869 | params rms: 0.060508 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:45:36.349744 | finish at 2026-10-17 18:04:21 + [2026-10-15 01:22:31] iteration 1328/ 2384 | consumed samples: 42496 | elapsed time per iteration (ms): 226029.1 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 74225.5 | valid tokens/s: 74225.5 | physical tokens/s: 75048.5 | loss tokens/s: 52940.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.323702E-01 | lm-global-supervised-tokens: 1.196602E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.549412E-02 | lm-token-buffer-mib: 2.351256E+02 | global_load_balancing_loss: 1.003040E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11174.834 | params rms: 0.060508 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:18:06.759247 | finish at 2026-10-17 19:40:37 + [2026-10-15 01:26:17] iteration 1329/ 2384 | consumed samples: 42528 | elapsed time per iteration (ms): 226135.7 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 74190.7 | valid tokens/s: 74190.7 | physical tokens/s: 74631.9 | loss tokens/s: 50127.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.695965E-01 | lm-global-supervised-tokens: 1.133549E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.867282E-02 | lm-token-buffer-mib: 2.334286E+02 | global_load_balancing_loss: 1.002077E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11174.792 | params rms: 0.060508 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:16:13.150659 | finish at 2026-10-17 19:42:30 + [2026-10-15 01:30:23] iteration 1330/ 2384 | consumed samples: 42560 | elapsed time per iteration (ms): 246360.2 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 68100.3 | valid tokens/s: 68100.3 | physical tokens/s: 68632.4 | loss tokens/s: 50638.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.040542E-01 | lm-global-supervised-tokens: 1.247525E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.473010E-02 | lm-token-buffer-mib: 2.354156E+02 | global_load_balancing_loss: 9.998072E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11174.752 | params rms: 0.060508 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:07:43.600466 | finish at 2026-10-18 01:38:07 + [2026-10-15 01:34:04] iteration 1331/ 2384 | consumed samples: 42592 | elapsed time per iteration (ms): 220501.8 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.52% | FLOPs estimator: workload-aware | logical tokens/s: 76086.4 | valid tokens/s: 76086.4 | physical tokens/s: 76472.2 | loss tokens/s: 57979.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.739375E-01 | lm-global-supervised-tokens: 1.278465E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.658291E-02 | lm-token-buffer-mib: 2.330018E+02 | global_load_balancing_loss: 9.984961E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11174.715 | params rms: 0.060507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:29:48.447683 | finish at 2026-10-17 18:03:52 + [2026-10-15 01:37:47] iteration 1332/ 2384 | consumed samples: 42624 | elapsed time per iteration (ms): 223884.3 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 74937.0 | valid tokens/s: 74937.0 | physical tokens/s: 75422.1 | loss tokens/s: 58107.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.209375E-01 | lm-global-supervised-tokens: 1.300939E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.578991E-02 | lm-token-buffer-mib: 2.336005E+02 | global_load_balancing_loss: 9.982142E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11174.678 | params rms: 0.060507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:25:26.268043 | finish at 2026-10-17 19:03:14 + [2026-10-15 01:41:29] iteration 1333/ 2384 | consumed samples: 42656 | elapsed time per iteration (ms): 221426.0 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 75768.6 | valid tokens/s: 75768.6 | physical tokens/s: 76142.7 | loss tokens/s: 57781.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.409469E-01 | lm-global-supervised-tokens: 1.279428E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.087177E-02 | lm-token-buffer-mib: 2.323432E+02 | global_load_balancing_loss: 9.984749E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11174.645 | params rms: 0.060507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:38:38.707832 | finish at 2026-10-17 18:20:08 + [2026-10-15 01:45:18] iteration 1334/ 2384 | consumed samples: 42688 | elapsed time per iteration (ms): 228853.6 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.99% | FLOPs estimator: workload-aware | logical tokens/s: 73309.6 | valid tokens/s: 73309.5 | physical tokens/s: 73901.8 | loss tokens/s: 55900.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.748140E-01 | lm-global-supervised-tokens: 1.279292E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.791022E-02 | lm-token-buffer-mib: 2.340150E+02 | global_load_balancing_loss: 9.998721E-01 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11174.614 | params rms: 0.060507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:44:56.326089 | finish at 2026-10-17 20:30:14 + [2026-10-15 01:49:02] iteration 1335/ 2384 | consumed samples: 42720 | elapsed time per iteration (ms): 224321.7 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.28% | FLOPs estimator: workload-aware | logical tokens/s: 74790.9 | valid tokens/s: 74790.9 | physical tokens/s: 75421.7 | loss tokens/s: 54595.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.535495E-01 | lm-global-supervised-tokens: 1.224701E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.276713E-02 | lm-token-buffer-mib: 2.344555E+02 | global_load_balancing_loss: 1.001301E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11174.582 | params rms: 0.060507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:21:53.419133 | finish at 2026-10-17 19:10:55 + [2026-10-15 01:52:45] iteration 1336/ 2384 | consumed samples: 42752 | elapsed time per iteration (ms): 222591.9 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.42% | FLOPs estimator: workload-aware | logical tokens/s: 75371.8 | valid tokens/s: 75371.8 | physical tokens/s: 76009.2 | loss tokens/s: 55764.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.876582E-01 | lm-global-supervised-tokens: 1.241262E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.325289E-02 | lm-token-buffer-mib: 2.340708E+02 | global_load_balancing_loss: 1.001109E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11174.546 | params rms: 0.060507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:47:56.309816 | finish at 2026-10-17 18:40:41 + [2026-10-15 01:56:25] iteration 1337/ 2384 | consumed samples: 42784 | elapsed time per iteration (ms): 219984.8 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 76265.3 | valid tokens/s: 76265.3 | physical tokens/s: 76639.8 | loss tokens/s: 55676.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.626391E-01 | lm-global-supervised-tokens: 1.224797E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.535417E-02 | lm-token-buffer-mib: 2.325831E+02 | global_load_balancing_loss: 1.001782E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11174.509 | params rms: 0.060506 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:58:44.099185 | finish at 2026-10-17 17:55:09 + [2026-10-15 02:00:06] iteration 1338/ 2384 | consumed samples: 42816 | elapsed time per iteration (ms): 221627.2 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.36% | FLOPs estimator: workload-aware | logical tokens/s: 75700.1 | valid tokens/s: 75700.1 | physical tokens/s: 76245.1 | loss tokens/s: 56565.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.059709E-01 | lm-global-supervised-tokens: 1.253647E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.210599E-02 | lm-token-buffer-mib: 2.333287E+02 | global_load_balancing_loss: 1.001648E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11174.474 | params rms: 0.060506 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:23:42.023401 | finish at 2026-10-17 18:23:48 + [2026-10-15 02:03:48] iteration 1339/ 2384 | consumed samples: 42848 | elapsed time per iteration (ms): 221840.3 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 75627.2 | valid tokens/s: 75627.2 | physical tokens/s: 76193.8 | loss tokens/s: 56384.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.065047E-01 | lm-global-supervised-tokens: 1.250840E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.760051E-02 | lm-token-buffer-mib: 2.330126E+02 | global_load_balancing_loss: 1.000039E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11174.442 | params rms: 0.060506 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:23:43.138502 | finish at 2026-10-17 18:27:31 + [2026-10-15 02:07:53] iteration 1340/ 2384 | consumed samples: 42880 | elapsed time per iteration (ms): 244690.6 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.24% | FLOPs estimator: workload-aware | logical tokens/s: 68565.0 | valid tokens/s: 68565.0 | physical tokens/s: 69388.9 | loss tokens/s: 48301.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.417865E-01 | lm-global-supervised-tokens: 1.181881E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.700114E-02 | lm-token-buffer-mib: 2.331118E+02 | global_load_balancing_loss: 1.000820E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11174.406 | params rms: 0.060506 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:57:36.984572 | finish at 2026-10-18 01:05:30 + [2026-10-15 02:11:34] iteration 1341/ 2384 | consumed samples: 42912 | elapsed time per iteration (ms): 220794.1 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 75985.7 | valid tokens/s: 75985.7 | physical tokens/s: 76166.4 | loss tokens/s: 55130.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.884736E-01 | lm-global-supervised-tokens: 1.217244E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.330115E-02 | lm-token-buffer-mib: 2.323324E+02 | global_load_balancing_loss: 9.996145E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11174.369 | params rms: 0.060506 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:58:08.240629 | finish at 2026-10-17 18:09:42 + [2026-10-15 02:15:17] iteration 1342/ 2384 | consumed samples: 42944 | elapsed time per iteration (ms): 223785.5 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.26% | FLOPs estimator: workload-aware | logical tokens/s: 74969.8 | valid tokens/s: 74969.8 | physical tokens/s: 75349.0 | loss tokens/s: 51554.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.186714E-01 | lm-global-supervised-tokens: 1.153712E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.447442E-02 | lm-token-buffer-mib: 2.327705E+02 | global_load_balancing_loss: 1.000246E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11174.333 | params rms: 0.060505 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:46:24.485338 | finish at 2026-10-17 19:01:42 + [2026-10-15 02:19:02] iteration 1343/ 2384 | consumed samples: 42976 | elapsed time per iteration (ms): 224326.6 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.32% | FLOPs estimator: workload-aware | logical tokens/s: 74789.1 | valid tokens/s: 74789.1 | physical tokens/s: 76493.9 | loss tokens/s: 58016.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.717284E-01 | lm-global-supervised-tokens: 1.301468E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.086033E-02 | lm-token-buffer-mib: 2.374071E+02 | global_load_balancing_loss: 9.998394E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11174.299 | params rms: 0.060505 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:52:03.987700 | finish at 2026-10-17 19:11:06 + [2026-10-15 02:22:43] iteration 1344/ 2384 | consumed samples: 43008 | elapsed time per iteration (ms): 221649.9 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 75692.2 | valid tokens/s: 75692.2 | physical tokens/s: 76446.6 | loss tokens/s: 59104.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.292914E-01 | lm-global-supervised-tokens: 1.310058E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.674093E-02 | lm-token-buffer-mib: 2.351181E+02 | global_load_balancing_loss: 9.992916E-01 | loss scale: 1.0 | grad norm: 0.276 | params norm: 11174.265 | params rms: 0.060505 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:01:55.911827 | finish at 2026-10-17 18:24:39 + [2026-10-15 02:26:31] iteration 1345/ 2384 | consumed samples: 43040 | elapsed time per iteration (ms): 227329.4 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 73801.3 | valid tokens/s: 73801.3 | physical tokens/s: 74401.8 | loss tokens/s: 54795.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.803680E-01 | lm-global-supervised-tokens: 1.245659E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.769125E-02 | lm-token-buffer-mib: 2.344278E+02 | global_load_balancing_loss: 1.001299E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11174.231 | params rms: 0.060505 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:36:35.275152 | finish at 2026-10-17 20:03:06 + [2026-10-15 02:30:12] iteration 1346/ 2384 | consumed samples: 43072 | elapsed time per iteration (ms): 220968.3 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 75925.8 | valid tokens/s: 75925.8 | physical tokens/s: 76580.8 | loss tokens/s: 59823.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.692812E-01 | lm-global-supervised-tokens: 1.321912E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.370442E-02 | lm-token-buffer-mib: 2.330796E+02 | global_load_balancing_loss: 1.000819E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11174.200 | params rms: 0.060505 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:42:45.117286 | finish at 2026-10-17 18:12:57 + [2026-10-15 02:33:56] iteration 1347/ 2384 | consumed samples: 43104 | elapsed time per iteration (ms): 224402.6 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 74763.8 | valid tokens/s: 74763.8 | physical tokens/s: 75254.2 | loss tokens/s: 55860.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.009840E-01 | lm-global-supervised-tokens: 1.253525E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.715902E-02 | lm-token-buffer-mib: 2.344370E+02 | global_load_balancing_loss: 1.001138E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11174.167 | params rms: 0.060504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:38:25.491060 | finish at 2026-10-17 19:12:22 + [2026-10-15 02:37:38] iteration 1348/ 2384 | consumed samples: 43136 | elapsed time per iteration (ms): 221558.9 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 75723.4 | valid tokens/s: 75723.4 | physical tokens/s: 75955.7 | loss tokens/s: 57144.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.793924E-01 | lm-global-supervised-tokens: 1.266092E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.662691E-02 | lm-token-buffer-mib: 2.328326E+02 | global_load_balancing_loss: 9.996552E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11174.133 | params rms: 0.060504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:45:35.072145 | finish at 2026-10-17 18:23:13 + [2026-10-15 02:41:20] iteration 1349/ 2384 | consumed samples: 43168 | elapsed time per iteration (ms): 222467.0 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 75414.4 | valid tokens/s: 75414.3 | physical tokens/s: 76070.0 | loss tokens/s: 58511.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.917353E-01 | lm-global-supervised-tokens: 1.301684E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.445179E-02 | lm-token-buffer-mib: 2.338013E+02 | global_load_balancing_loss: 1.001964E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11174.099 | params rms: 0.060504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:57:33.310461 | finish at 2026-10-17 18:38:53 + [2026-10-15 02:45:26] iteration 1350/ 2384 | consumed samples: 43200 | elapsed time per iteration (ms): 245449.6 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 68352.9 | valid tokens/s: 68352.9 | physical tokens/s: 69103.9 | loss tokens/s: 56576.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.965923E-01 | lm-global-supervised-tokens: 1.388665E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.675910E-02 | lm-token-buffer-mib: 2.356849E+02 | global_load_balancing_loss: 9.995879E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11174.065 | params rms: 0.060504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:29:54.924099 | finish at 2026-10-18 01:15:20 + [2026-10-15 02:49:08] iteration 1351/ 2384 | consumed samples: 43232 | elapsed time per iteration (ms): 222693.3 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 75337.7 | valid tokens/s: 75337.7 | physical tokens/s: 75776.1 | loss tokens/s: 53473.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.611274E-01 | lm-global-supervised-tokens: 1.190826E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.278993E-02 | lm-token-buffer-mib: 2.335067E+02 | global_load_balancing_loss: 9.997407E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11174.028 | params rms: 0.060504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:54:02.138764 | finish at 2026-10-17 18:43:10 + [2026-10-15 02:52:51] iteration 1352/ 2384 | consumed samples: 43264 | elapsed time per iteration (ms): 222508.9 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.92% | FLOPs estimator: workload-aware | logical tokens/s: 75400.1 | valid tokens/s: 75400.1 | physical tokens/s: 76247.8 | loss tokens/s: 59393.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.336757E-01 | lm-global-supervised-tokens: 1.321552E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.370580E-02 | lm-token-buffer-mib: 2.327349E+02 | global_load_balancing_loss: 9.999384E-01 | loss scale: 1.0 | grad norm: 0.253 | params norm: 11173.991 | params rms: 0.060504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:47:09.209084 | finish at 2026-10-17 18:40:00 + [2026-10-15 02:56:31] iteration 1353/ 2384 | consumed samples: 43296 | elapsed time per iteration (ms): 219914.4 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.16% | FLOPs estimator: workload-aware | logical tokens/s: 76289.5 | valid tokens/s: 76289.5 | physical tokens/s: 77274.6 | loss tokens/s: 60536.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.160543E-01 | lm-global-supervised-tokens: 1.331278E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.767663E-02 | lm-token-buffer-mib: 2.337880E+02 | global_load_balancing_loss: 9.994514E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11173.948 | params rms: 0.060503 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:58:51.723398 | finish at 2026-10-17 17:55:22 + [2026-10-15 03:00:13] iteration 1354/ 2384 | consumed samples: 43328 | elapsed time per iteration (ms): 222437.7 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75424.3 | valid tokens/s: 75424.3 | physical tokens/s: 76171.5 | loss tokens/s: 53623.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.538190E-01 | lm-global-supervised-tokens: 1.192779E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.937989E-02 | lm-token-buffer-mib: 2.342800E+02 | global_load_balancing_loss: 9.999251E-01 | loss scale: 1.0 | grad norm: 0.488 | params norm: 11173.907 | params rms: 0.060503 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:38:30.871308 | finish at 2026-10-17 18:38:44 + [2026-10-15 03:03:57] iteration 1355/ 2384 | consumed samples: 43360 | elapsed time per iteration (ms): 223821.9 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 74957.8 | valid tokens/s: 74957.8 | physical tokens/s: 75367.9 | loss tokens/s: 53320.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.154784E-01 | lm-global-supervised-tokens: 1.193430E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.006813E-02 | lm-token-buffer-mib: 2.335209E+02 | global_load_balancing_loss: 9.994422E-01 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11173.869 | params rms: 0.060503 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:58:32.741857 | finish at 2026-10-17 19:02:30 + [2026-10-15 03:07:40] iteration 1356/ 2384 | consumed samples: 43392 | elapsed time per iteration (ms): 222983.1 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 75239.7 | valid tokens/s: 75239.7 | physical tokens/s: 76114.1 | loss tokens/s: 57620.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.402177E-01 | lm-global-supervised-tokens: 1.284831E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.472212E-02 | lm-token-buffer-mib: 2.359106E+02 | global_load_balancing_loss: 1.001122E+00 | loss scale: 1.0 | grad norm: 0.669 | params norm: 11173.832 | params rms: 0.060503 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:40:26.646098 | finish at 2026-10-17 18:48:07 + [2026-10-15 03:11:23] iteration 1357/ 2384 | consumed samples: 43424 | elapsed time per iteration (ms): 223213.8 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 75162.0 | valid tokens/s: 75162.0 | physical tokens/s: 75950.0 | loss tokens/s: 59640.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.852907E-01 | lm-global-supervised-tokens: 1.331264E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.404932E-02 | lm-token-buffer-mib: 2.334046E+02 | global_load_balancing_loss: 1.002902E+00 | loss scale: 1.0 | grad norm: 0.288 | params norm: 11173.794 | params rms: 0.060502 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:40:40.529458 | finish at 2026-10-17 18:52:04 + [2026-10-15 03:15:04] iteration 1358/ 2384 | consumed samples: 43456 | elapsed time per iteration (ms): 220989.2 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75918.6 | valid tokens/s: 75918.6 | physical tokens/s: 76655.8 | loss tokens/s: 60702.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.241675E-01 | lm-global-supervised-tokens: 1.341452E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.284406E-02 | lm-token-buffer-mib: 2.350590E+02 | global_load_balancing_loss: 1.000199E+00 | loss scale: 1.0 | grad norm: 0.547 | params norm: 11173.753 | params rms: 0.060502 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:58:54.944269 | finish at 2026-10-17 18:13:59 + [2026-10-15 03:18:50] iteration 1359/ 2384 | consumed samples: 43488 | elapsed time per iteration (ms): 225987.1 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 74239.5 | valid tokens/s: 74239.5 | physical tokens/s: 75007.2 | loss tokens/s: 55109.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.449207E-01 | lm-global-supervised-tokens: 1.245404E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.390692E-02 | lm-token-buffer-mib: 2.332833E+02 | global_load_balancing_loss: 1.000884E+00 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11173.713 | params rms: 0.060502 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:20:36.792046 | finish at 2026-10-17 19:39:27 + [2026-10-15 03:22:53] iteration 1360/ 2384 | consumed samples: 43520 | elapsed time per iteration (ms): 243015.0 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.16% | FLOPs estimator: workload-aware | logical tokens/s: 69037.6 | valid tokens/s: 69037.6 | physical tokens/s: 69298.5 | loss tokens/s: 52183.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.136057E-01 | lm-global-supervised-tokens: 1.268145E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.944521E-02 | lm-token-buffer-mib: 2.330587E+02 | global_load_balancing_loss: 1.000546E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11173.675 | params rms: 0.060502 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:07:27.312500 | finish at 2026-10-18 00:30:20 +times across ranks (ms): + save-checkpoint: + rank 0: 24894.26 + rank 1: 24894.28 + rank 2: 24894.27 + rank 3: 24894.27 + rank 4: 24894.26 + rank 5: 24894.32 + rank 6: 24894.31 + rank 7: 24894.27 + rank 8: 24894.39 + rank 9: 24894.40 + rank 10: 24894.40 + rank 11: 24894.40 + rank 12: 24894.42 + rank 13: 24894.38 + rank 14: 24894.38 + rank 15: 24894.39 + rank 16: 24896.37 + rank 17: 24896.34 + rank 18: 24896.37 + rank 19: 24896.34 + rank 20: 24896.42 + rank 21: 24896.34 + rank 22: 24896.36 + rank 23: 24896.42 + rank 24: 24894.41 + rank 25: 24894.41 + rank 26: 24894.38 + rank 27: 24894.37 + rank 28: 24894.42 + rank 29: 24894.44 + rank 30: 24894.41 + rank 31: 24894.44 + rank 32: 24894.01 + rank 33: 24894.04 + rank 34: 24894.00 + rank 35: 24894.00 + rank 36: 24893.98 + rank 37: 24894.03 + rank 38: 24893.97 + rank 39: 24893.99 + rank 40: 24894.03 + rank 41: 24894.03 + rank 42: 24894.00 + rank 43: 24893.97 + rank 44: 24893.98 + rank 45: 24893.79 + rank 46: 24893.98 + rank 47: 24893.98 + rank 48: 24894.71 + rank 49: 24894.76 + rank 50: 24894.75 + rank 51: 24894.71 + rank 52: 24894.73 + rank 53: 24894.73 + rank 54: 24894.71 + rank 55: 24894.71 + rank 56: 24894.79 + rank 57: 24894.77 + rank 58: 24894.77 + rank 59: 24894.82 + rank 60: 24894.75 + rank 61: 24894.77 + rank 62: 24894.77 + rank 63: 24894.76 + [2026-10-15 03:27:19] iteration 1361/ 2384 | consumed samples: 43552 | elapsed time per iteration (ms): 241079.2 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 69592.0 | valid tokens/s: 69592.0 | physical tokens/s: 69942.8 | loss tokens/s: 52437.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.215577E-01 | lm-global-supervised-tokens: 1.264162E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.188102E-01 | lm-token-buffer-mib: 2.328470E+02 | global_load_balancing_loss: 9.998307E-01 | loss scale: 1.0 | grad norm: 0.299 | params norm: 11173.637 | params rms: 0.060502 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:30:24.010167 | finish at 2026-10-17 23:57:43 + [2026-10-15 03:31:03] iteration 1362/ 2384 | consumed samples: 43584 | elapsed time per iteration (ms): 223895.8 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.97% | FLOPs estimator: workload-aware | logical tokens/s: 74933.0 | valid tokens/s: 74933.0 | physical tokens/s: 75512.8 | loss tokens/s: 56436.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.635044E-01 | lm-global-supervised-tokens: 1.263579E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.940023E-02 | lm-token-buffer-mib: 2.330571E+02 | global_load_balancing_loss: 9.991987E-01 | loss scale: 1.0 | grad norm: 0.213 | params norm: 11173.601 | params rms: 0.060501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:33:41.518925 | finish at 2026-10-17 19:04:45 + [2026-10-15 03:34:49] iteration 1363/ 2384 | consumed samples: 43616 | elapsed time per iteration (ms): 225717.3 | throughput per GPU (TFLOP/s/GPU): 27.2 | MFU 8.72% | FLOPs estimator: workload-aware | logical tokens/s: 74328.2 | valid tokens/s: 74328.2 | physical tokens/s: 75083.2 | loss tokens/s: 55512.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.648263E-01 | lm-global-supervised-tokens: 1.253005E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.908307E-02 | lm-token-buffer-mib: 2.363022E+02 | global_load_balancing_loss: 9.995334E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11173.561 | params rms: 0.060501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:00:57.399994 | finish at 2026-10-17 19:35:46 + [2026-10-15 03:38:29] iteration 1364/ 2384 | consumed samples: 43648 | elapsed time per iteration (ms): 220248.6 | throughput per GPU (TFLOP/s/GPU): 27.0 | MFU 8.66% | FLOPs estimator: workload-aware | logical tokens/s: 76173.7 | valid tokens/s: 76173.7 | physical tokens/s: 76544.2 | loss tokens/s: 56415.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.897565E-01 | lm-global-supervised-tokens: 1.242541E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.359088E-02 | lm-token-buffer-mib: 2.326858E+02 | global_load_balancing_loss: 9.998620E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11173.519 | params rms: 0.060501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:24:13.531880 | finish at 2026-10-17 18:02:43 + [2026-10-15 03:42:12] iteration 1365/ 2384 | consumed samples: 43680 | elapsed time per iteration (ms): 222837.9 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.70% | FLOPs estimator: workload-aware | logical tokens/s: 75288.7 | valid tokens/s: 75288.7 | physical tokens/s: 75981.0 | loss tokens/s: 60371.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.720353E-01 | lm-global-supervised-tokens: 1.345306E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.520604E-02 | lm-token-buffer-mib: 2.353610E+02 | global_load_balancing_loss: 1.001240E+00 | loss scale: 1.0 | grad norm: 0.172 | params norm: 11173.478 | params rms: 0.060501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:04:31.837757 | finish at 2026-10-17 18:46:44 + [2026-10-15 03:45:57] iteration 1366/ 2384 | consumed samples: 43712 | elapsed time per iteration (ms): 224854.7 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.87% | FLOPs estimator: workload-aware | logical tokens/s: 74613.5 | valid tokens/s: 74613.5 | physical tokens/s: 75229.3 | loss tokens/s: 58424.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.484716E-01 | lm-global-supervised-tokens: 1.313690E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.766376E-02 | lm-token-buffer-mib: 2.354095E+02 | global_load_balancing_loss: 1.000129E+00 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11173.437 | params rms: 0.060501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:35:02.050953 | finish at 2026-10-17 19:20:59 + [2026-10-15 03:49:39] iteration 1367/ 2384 | consumed samples: 43744 | elapsed time per iteration (ms): 222164.5 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75516.9 | valid tokens/s: 75516.9 | physical tokens/s: 76730.7 | loss tokens/s: 61450.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.977773E-01 | lm-global-supervised-tokens: 1.365218E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.375826E-02 | lm-token-buffer-mib: 2.350686E+02 | global_load_balancing_loss: 1.002572E+00 | loss scale: 1.0 | grad norm: 0.284 | params norm: 11173.394 | params rms: 0.060500 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:45:41.316381 | finish at 2026-10-17 18:35:20 + [2026-10-15 03:53:22] iteration 1368/ 2384 | consumed samples: 43776 | elapsed time per iteration (ms): 223229.6 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 75156.6 | valid tokens/s: 75156.6 | physical tokens/s: 75836.0 | loss tokens/s: 55508.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.420509E-01 | lm-global-supervised-tokens: 1.239116E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.896137E-02 | lm-token-buffer-mib: 2.327231E+02 | global_load_balancing_loss: 1.001309E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11173.351 | params rms: 0.060500 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:00:01.305527 | finish at 2026-10-17 18:53:23 + [2026-10-15 03:57:04] iteration 1369/ 2384 | consumed samples: 43808 | elapsed time per iteration (ms): 221759.8 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75654.7 | valid tokens/s: 75654.7 | physical tokens/s: 76497.1 | loss tokens/s: 59342.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.337918E-01 | lm-global-supervised-tokens: 1.315970E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.351155E-02 | lm-token-buffer-mib: 2.360365E+02 | global_load_balancing_loss: 1.000581E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11173.310 | params rms: 0.060500 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:31:26.159689 | finish at 2026-10-17 18:28:30 + [2026-10-15 04:01:07] iteration 1370/ 2384 | consumed samples: 43840 | elapsed time per iteration (ms): 243372.9 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 68936.1 | valid tokens/s: 68936.1 | physical tokens/s: 69411.7 | loss tokens/s: 50462.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.468635E-01 | lm-global-supervised-tokens: 1.228120E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.294469E-02 | lm-token-buffer-mib: 2.338495E+02 | global_load_balancing_loss: 1.000412E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11173.272 | params rms: 0.060500 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:33:00.077335 | finish at 2026-10-18 00:34:07 + [2026-10-15 04:04:49] iteration 1371/ 2384 | consumed samples: 43872 | elapsed time per iteration (ms): 222225.0 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 75496.5 | valid tokens/s: 75496.5 | physical tokens/s: 76247.9 | loss tokens/s: 57619.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.276688E-01 | lm-global-supervised-tokens: 1.280442E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.908706E-02 | lm-token-buffer-mib: 2.343177E+02 | global_load_balancing_loss: 1.000531E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11173.238 | params rms: 0.060499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:31:53.940844 | finish at 2026-10-17 18:36:43 + [2026-10-15 04:08:32] iteration 1372/ 2384 | consumed samples: 43904 | elapsed time per iteration (ms): 222414.5 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 75432.0 | valid tokens/s: 75432.0 | physical tokens/s: 76309.3 | loss tokens/s: 56621.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.218849E-01 | lm-global-supervised-tokens: 1.259349E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.286239E-02 | lm-token-buffer-mib: 2.358045E+02 | global_load_balancing_loss: 9.994797E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11173.206 | params rms: 0.060499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:31:23.519359 | finish at 2026-10-17 18:39:55 + [2026-10-15 04:12:19] iteration 1373/ 2384 | consumed samples: 43936 | elapsed time per iteration (ms): 226955.4 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 73923.0 | valid tokens/s: 73922.9 | physical tokens/s: 75279.9 | loss tokens/s: 59756.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.146881E-01 | lm-global-supervised-tokens: 1.356201E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.407537E-02 | lm-token-buffer-mib: 2.370708E+02 | global_load_balancing_loss: 1.000170E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11173.178 | params rms: 0.060499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:44:11.924576 | finish at 2026-10-17 19:56:31 + [2026-10-15 04:16:03] iteration 1374/ 2384 | consumed samples: 43968 | elapsed time per iteration (ms): 223800.1 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 74965.2 | valid tokens/s: 74965.2 | physical tokens/s: 75322.9 | loss tokens/s: 56380.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.531268E-01 | lm-global-supervised-tokens: 1.261801E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.066247E-02 | lm-token-buffer-mib: 2.324407E+02 | global_load_balancing_loss: 1.000933E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11173.151 | params rms: 0.060499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:47:18.066654 | finish at 2026-10-17 19:03:21 + [2026-10-15 04:19:48] iteration 1375/ 2384 | consumed samples: 44000 | elapsed time per iteration (ms): 225604.8 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 74365.4 | valid tokens/s: 74365.4 | physical tokens/s: 75453.1 | loss tokens/s: 55884.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.542847E-01 | lm-global-supervised-tokens: 1.260791E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.619287E-02 | lm-token-buffer-mib: 2.333170E+02 | global_load_balancing_loss: 1.001319E+00 | loss scale: 1.0 | grad norm: 0.183 | params norm: 11173.121 | params rms: 0.060499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:13:55.260988 | finish at 2026-10-17 19:33:43 + [2026-10-15 04:23:31] iteration 1376/ 2384 | consumed samples: 44032 | elapsed time per iteration (ms): 222535.9 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 75391.0 | valid tokens/s: 75391.0 | physical tokens/s: 75961.6 | loss tokens/s: 52481.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.669555E-01 | lm-global-supervised-tokens: 1.167892E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.106304E-02 | lm-token-buffer-mib: 2.345094E+02 | global_load_balancing_loss: 9.996012E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11173.089 | params rms: 0.060499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:18:36.206543 | finish at 2026-10-17 18:42:07 + [2026-10-15 04:27:11] iteration 1377/ 2384 | consumed samples: 44064 | elapsed time per iteration (ms): 220526.7 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 76077.9 | valid tokens/s: 76077.9 | physical tokens/s: 76578.0 | loss tokens/s: 53447.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.814720E-01 | lm-global-supervised-tokens: 1.178657E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.130437E-02 | lm-token-buffer-mib: 2.331847E+02 | global_load_balancing_loss: 1.000107E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11173.054 | params rms: 0.060498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:41:10.405647 | finish at 2026-10-17 18:08:22 + [2026-10-15 04:30:54] iteration 1378/ 2384 | consumed samples: 44096 | elapsed time per iteration (ms): 222460.9 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 75416.4 | valid tokens/s: 75416.4 | physical tokens/s: 76022.1 | loss tokens/s: 54182.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.658497E-01 | lm-global-supervised-tokens: 1.205356E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.895193E-02 | lm-token-buffer-mib: 2.343060E+02 | global_load_balancing_loss: 9.996681E-01 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11173.021 | params rms: 0.060498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:09:55.647000 | finish at 2026-10-17 18:40:49 + [2026-10-15 04:34:40] iteration 1379/ 2384 | consumed samples: 44128 | elapsed time per iteration (ms): 226199.3 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 74169.9 | valid tokens/s: 74169.9 | physical tokens/s: 75805.4 | loss tokens/s: 58872.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.669855E-01 | lm-global-supervised-tokens: 1.331687E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.741751E-02 | lm-token-buffer-mib: 2.391747E+02 | global_load_balancing_loss: 9.998946E-01 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11172.994 | params rms: 0.060498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:08:50.335368 | finish at 2026-10-17 19:43:30 + [2026-10-15 04:38:41] iteration 1380/ 2384 | consumed samples: 44160 | elapsed time per iteration (ms): 241150.0 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 69571.6 | valid tokens/s: 69571.6 | physical tokens/s: 70372.4 | loss tokens/s: 55899.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.147809E-01 | lm-global-supervised-tokens: 1.348010E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.212333E-02 | lm-token-buffer-mib: 2.337803E+02 | global_load_balancing_loss: 1.002000E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11172.970 | params rms: 0.060498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:15:14.564429 | finish at 2026-10-17 23:53:56 + [2026-10-15 04:42:24] iteration 1381/ 2384 | consumed samples: 44192 | elapsed time per iteration (ms): 222969.1 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 75244.4 | valid tokens/s: 75244.4 | physical tokens/s: 75829.8 | loss tokens/s: 60114.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.468270E-01 | lm-global-supervised-tokens: 1.340364E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.725868E-02 | lm-token-buffer-mib: 2.338376E+02 | global_load_balancing_loss: 1.001258E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11172.945 | params rms: 0.060498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:07:17.994146 | finish at 2026-10-17 18:49:42 + [2026-10-15 04:46:07] iteration 1382/ 2384 | consumed samples: 44224 | elapsed time per iteration (ms): 222504.8 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75401.5 | valid tokens/s: 75401.5 | physical tokens/s: 76655.4 | loss tokens/s: 61061.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.308783E-01 | lm-global-supervised-tokens: 1.358655E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.681557E-02 | lm-token-buffer-mib: 2.362805E+02 | global_load_balancing_loss: 1.000137E+00 | loss scale: 1.0 | grad norm: 0.314 | params norm: 11172.915 | params rms: 0.060498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:55:49.768591 | finish at 2026-10-17 18:41:56 + [2026-10-15 04:49:52] iteration 1383/ 2384 | consumed samples: 44256 | elapsed time per iteration (ms): 225872.4 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 74277.2 | valid tokens/s: 74277.2 | physical tokens/s: 75027.4 | loss tokens/s: 54209.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.035193E-01 | lm-global-supervised-tokens: 1.224434E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.249740E-02 | lm-token-buffer-mib: 2.354229E+02 | global_load_balancing_loss: 1.001508E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11172.886 | params rms: 0.060498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:48:18.275309 | finish at 2026-10-17 19:38:11 + [2026-10-15 04:53:37] iteration 1384/ 2384 | consumed samples: 44288 | elapsed time per iteration (ms): 224897.1 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 74599.4 | valid tokens/s: 74599.4 | physical tokens/s: 74989.4 | loss tokens/s: 54615.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.176722E-01 | lm-global-supervised-tokens: 1.228281E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.318817E-02 | lm-token-buffer-mib: 2.335167E+02 | global_load_balancing_loss: 9.991278E-01 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11172.854 | params rms: 0.060497 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:28:17.135496 | finish at 2026-10-17 19:21:54 + [2026-10-15 04:57:18] iteration 1385/ 2384 | consumed samples: 44320 | elapsed time per iteration (ms): 221144.6 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75865.3 | valid tokens/s: 75865.3 | physical tokens/s: 76572.4 | loss tokens/s: 58705.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.014125E-01 | lm-global-supervised-tokens: 1.298232E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.976335E-02 | lm-token-buffer-mib: 2.347643E+02 | global_load_balancing_loss: 9.993963E-01 | loss scale: 1.0 | grad norm: 0.326 | params norm: 11172.823 | params rms: 0.060497 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:22:03.446502 | finish at 2026-10-17 18:19:22 + [2026-10-15 05:00:59] iteration 1386/ 2384 | consumed samples: 44352 | elapsed time per iteration (ms): 220399.8 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 76121.5 | valid tokens/s: 76121.5 | physical tokens/s: 76946.7 | loss tokens/s: 59050.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.627721E-01 | lm-global-supervised-tokens: 1.301477E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.698583E-02 | lm-token-buffer-mib: 2.354031E+02 | global_load_balancing_loss: 9.998385E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11172.790 | params rms: 0.060497 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:05:59.012359 | finish at 2026-10-17 18:06:58 + [2026-10-15 05:04:43] iteration 1387/ 2384 | consumed samples: 44384 | elapsed time per iteration (ms): 223899.1 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 74931.9 | valid tokens/s: 74931.9 | physical tokens/s: 75357.6 | loss tokens/s: 54271.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.069954E-01 | lm-global-supervised-tokens: 1.215139E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.918732E-02 | lm-token-buffer-mib: 2.334402E+02 | global_load_balancing_loss: 1.000123E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11172.756 | params rms: 0.060497 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:00:27.430101 | finish at 2026-10-17 19:05:10 + [2026-10-15 05:08:30] iteration 1388/ 2384 | consumed samples: 44416 | elapsed time per iteration (ms): 227690.5 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 73684.0 | valid tokens/s: 73684.0 | physical tokens/s: 74425.8 | loss tokens/s: 55357.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.456009E-01 | lm-global-supervised-tokens: 1.260446E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.283969E-02 | lm-token-buffer-mib: 2.347693E+02 | global_load_balancing_loss: 1.000248E+00 | loss scale: 1.0 | grad norm: 0.172 | params norm: 11172.720 | params rms: 0.060497 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:59:39.748944 | finish at 2026-10-17 20:08:10 + [2026-10-15 05:12:12] iteration 1389/ 2384 | consumed samples: 44448 | elapsed time per iteration (ms): 221814.2 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75636.2 | valid tokens/s: 75636.2 | physical tokens/s: 76699.3 | loss tokens/s: 58395.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.006677E-01 | lm-global-supervised-tokens: 1.295300E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.513081E-02 | lm-token-buffer-mib: 2.352485E+02 | global_load_balancing_loss: 1.001233E+00 | loss scale: 1.0 | grad norm: 0.375 | params norm: 11172.681 | params rms: 0.060496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:18:25.100458 | finish at 2026-10-17 18:30:37 + [2026-10-15 05:16:17] iteration 1390/ 2384 | consumed samples: 44480 | elapsed time per iteration (ms): 244868.1 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 68515.1 | valid tokens/s: 68515.1 | physical tokens/s: 68736.4 | loss tokens/s: 45939.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.618429E-01 | lm-global-supervised-tokens: 1.124908E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.933844E-02 | lm-token-buffer-mib: 2.327271E+02 | global_load_balancing_loss: 1.001224E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11172.642 | params rms: 0.060496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:36:38.852699 | finish at 2026-10-18 00:52:56 + [2026-10-15 05:19:57] iteration 1391/ 2384 | consumed samples: 44512 | elapsed time per iteration (ms): 220282.2 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 76162.2 | valid tokens/s: 76162.2 | physical tokens/s: 76544.2 | loss tokens/s: 54110.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.694156E-01 | lm-global-supervised-tokens: 1.191947E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.232258E-02 | lm-token-buffer-mib: 2.330801E+02 | global_load_balancing_loss: 1.001411E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11172.608 | params rms: 0.060496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:45:40.191316 | finish at 2026-10-17 18:05:38 + [2026-10-15 05:23:42] iteration 1392/ 2384 | consumed samples: 44544 | elapsed time per iteration (ms): 224602.9 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 74697.0 | valid tokens/s: 74697.0 | physical tokens/s: 75613.4 | loss tokens/s: 56832.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.074314E-01 | lm-global-supervised-tokens: 1.276484E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.905955E-02 | lm-token-buffer-mib: 2.363974E+02 | global_load_balancing_loss: 1.001004E+00 | loss scale: 1.0 | grad norm: 0.261 | params norm: 11172.572 | params rms: 0.060496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:53:26.047501 | finish at 2026-10-17 19:17:08 + [2026-10-15 05:27:24] iteration 1393/ 2384 | consumed samples: 44576 | elapsed time per iteration (ms): 222108.6 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.40% | FLOPs estimator: workload-aware | logical tokens/s: 75536.0 | valid tokens/s: 75536.0 | physical tokens/s: 75858.2 | loss tokens/s: 61614.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.570716E-01 | lm-global-supervised-tokens: 1.368521E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.750469E-02 | lm-token-buffer-mib: 2.330004E+02 | global_load_balancing_loss: 9.997817E-01 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11172.535 | params rms: 0.060496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:08:29.664559 | finish at 2026-10-17 18:35:54 + [2026-10-15 05:31:11] iteration 1394/ 2384 | consumed samples: 44608 | elapsed time per iteration (ms): 226469.0 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.36% | FLOPs estimator: workload-aware | logical tokens/s: 74081.7 | valid tokens/s: 74081.7 | physical tokens/s: 75358.8 | loss tokens/s: 53481.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.549878E-01 | lm-global-supervised-tokens: 1.211183E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.433506E-02 | lm-token-buffer-mib: 2.394083E+02 | global_load_balancing_loss: 9.997303E-01 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11172.495 | params rms: 0.060495 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:16:44.287913 | finish at 2026-10-17 19:47:55 + [2026-10-15 05:34:52] iteration 1395/ 2384 | consumed samples: 44640 | elapsed time per iteration (ms): 221394.2 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.06% | FLOPs estimator: workload-aware | logical tokens/s: 75779.6 | valid tokens/s: 75779.6 | physical tokens/s: 77133.6 | loss tokens/s: 59788.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.063993E-01 | lm-global-supervised-tokens: 1.323678E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.528396E-02 | lm-token-buffer-mib: 2.357366E+02 | global_load_balancing_loss: 9.996108E-01 | loss scale: 1.0 | grad norm: 0.495 | params norm: 11172.458 | params rms: 0.060495 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:49:18.902320 | finish at 2026-10-17 18:24:11 + [2026-10-15 05:38:32] iteration 1396/ 2384 | consumed samples: 44672 | elapsed time per iteration (ms): 219965.3 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.13% | FLOPs estimator: workload-aware | logical tokens/s: 76272.0 | valid tokens/s: 76271.9 | physical tokens/s: 76761.5 | loss tokens/s: 56558.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.866529E-01 | lm-global-supervised-tokens: 1.244092E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.683420E-02 | lm-token-buffer-mib: 2.332464E+02 | global_load_balancing_loss: 9.995511E-01 | loss scale: 1.0 | grad norm: 0.329 | params norm: 11172.425 | params rms: 0.060495 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:22:05.730616 | finish at 2026-10-17 18:00:38 + [2026-10-15 05:42:14] iteration 1397/ 2384 | consumed samples: 44704 | elapsed time per iteration (ms): 221988.8 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75576.7 | valid tokens/s: 75576.7 | physical tokens/s: 77289.1 | loss tokens/s: 55794.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.413373E-01 | lm-global-supervised-tokens: 1.238581E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.596578E-02 | lm-token-buffer-mib: 2.336735E+02 | global_load_balancing_loss: 1.000401E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11172.390 | params rms: 0.060495 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:51:42.917881 | finish at 2026-10-17 18:33:57 + [2026-10-15 05:45:55] iteration 1398/ 2384 | consumed samples: 44736 | elapsed time per iteration (ms): 220808.3 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 75980.9 | valid tokens/s: 75980.9 | physical tokens/s: 76364.7 | loss tokens/s: 54910.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.137965E-01 | lm-global-supervised-tokens: 1.212475E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.176763E-02 | lm-token-buffer-mib: 2.338250E+02 | global_load_balancing_loss: 9.993419E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11172.352 | params rms: 0.060495 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:28:37.004505 | finish at 2026-10-17 18:14:32 + [2026-10-15 05:49:35] iteration 1399/ 2384 | consumed samples: 44768 | elapsed time per iteration (ms): 220487.0 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.07% | FLOPs estimator: workload-aware | logical tokens/s: 76091.5 | valid tokens/s: 76091.5 | physical tokens/s: 76544.7 | loss tokens/s: 58243.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.415318E-01 | lm-global-supervised-tokens: 1.284190E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.490177E-02 | lm-token-buffer-mib: 2.330976E+02 | global_load_balancing_loss: 1.000636E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11172.316 | params rms: 0.060494 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:19:39.695224 | finish at 2026-10-17 18:09:15 + [2026-10-15 05:53:39] iteration 1400/ 2384 | consumed samples: 44800 | elapsed time per iteration (ms): 243595.0 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 68873.2 | valid tokens/s: 68873.2 | physical tokens/s: 69294.3 | loss tokens/s: 50496.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.892428E-01 | lm-global-supervised-tokens: 1.230062E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.541850E-02 | lm-token-buffer-mib: 2.328376E+02 | global_load_balancing_loss: 1.000977E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11172.278 | params rms: 0.060494 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:34:57.480732 | finish at 2026-10-18 00:28:36 +times across ranks (ms): + save-checkpoint: + rank 0: 25016.79 + rank 1: 25016.84 + rank 2: 25016.82 + rank 3: 25016.82 + rank 4: 25016.69 + rank 5: 25016.82 + rank 6: 25016.80 + rank 7: 25016.79 + rank 8: 25014.98 + rank 9: 25015.02 + rank 10: 25014.97 + rank 11: 25014.94 + rank 12: 25014.96 + rank 13: 25014.94 + rank 14: 25014.96 + rank 15: 25014.95 + rank 16: 25015.50 + rank 17: 25015.49 + rank 18: 25015.52 + rank 19: 25015.50 + rank 20: 25015.52 + rank 21: 25015.48 + rank 22: 25015.48 + rank 23: 25015.46 + rank 24: 25015.65 + rank 25: 25015.65 + rank 26: 25015.65 + rank 27: 25015.58 + rank 28: 25015.56 + rank 29: 25016.52 + rank 30: 25015.65 + rank 31: 25015.59 + rank 32: 25015.46 + rank 33: 25015.49 + rank 34: 25015.40 + rank 35: 25015.45 + rank 36: 25015.42 + rank 37: 25015.41 + rank 38: 25015.34 + rank 39: 25015.35 + rank 40: 25015.46 + rank 41: 25015.49 + rank 42: 25015.46 + rank 43: 25015.47 + rank 44: 25015.45 + rank 45: 25015.44 + rank 46: 25015.43 + rank 47: 25015.47 + rank 48: 25014.95 + rank 49: 25014.99 + rank 50: 25014.84 + rank 51: 25015.01 + rank 52: 25015.07 + rank 53: 25015.09 + rank 54: 25014.92 + rank 55: 25014.95 + rank 56: 25015.46 + rank 57: 25015.47 + rank 58: 25015.48 + rank 59: 25015.48 + rank 60: 25015.47 + rank 61: 25015.46 + rank 62: 25015.44 + rank 63: 25015.44 + [2026-10-15 05:57:51] iteration 1401/ 2384 | consumed samples: 44832 | elapsed time per iteration (ms): 226873.1 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 73949.8 | valid tokens/s: 73949.8 | physical tokens/s: 74503.5 | loss tokens/s: 53614.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.665849E-01 | lm-global-supervised-tokens: 1.216363E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.118765E-01 | lm-token-buffer-mib: 2.335256E+02 | global_load_balancing_loss: 1.002159E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11172.238 | params rms: 0.060494 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:56:56.231561 | finish at 2026-10-17 19:54:47 + [2026-10-15 06:01:42] iteration 1402/ 2384 | consumed samples: 44864 | elapsed time per iteration (ms): 230799.6 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 72691.7 | valid tokens/s: 72691.7 | physical tokens/s: 73477.6 | loss tokens/s: 57242.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.673315E-01 | lm-global-supervised-tokens: 1.321164E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.540672E-02 | lm-token-buffer-mib: 2.358122E+02 | global_load_balancing_loss: 1.001511E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11172.199 | params rms: 0.060494 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:57:25.224648 | finish at 2026-10-17 20:59:07 + [2026-10-15 06:05:22] iteration 1403/ 2384 | consumed samples: 44896 | elapsed time per iteration (ms): 220206.9 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 76188.3 | valid tokens/s: 76188.3 | physical tokens/s: 76955.7 | loss tokens/s: 59139.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.407120E-01 | lm-global-supervised-tokens: 1.302287E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.547882E-02 | lm-token-buffer-mib: 2.337328E+02 | global_load_balancing_loss: 9.998779E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11172.161 | params rms: 0.060494 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:00:23.006907 | finish at 2026-10-17 18:05:45 + [2026-10-15 06:09:04] iteration 1404/ 2384 | consumed samples: 44928 | elapsed time per iteration (ms): 222468.6 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.81% | FLOPs estimator: workload-aware | logical tokens/s: 75413.7 | valid tokens/s: 75413.7 | physical tokens/s: 76414.4 | loss tokens/s: 59490.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.320514E-01 | lm-global-supervised-tokens: 1.323478E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.325675E-02 | lm-token-buffer-mib: 2.353416E+02 | global_load_balancing_loss: 1.000211E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11172.125 | params rms: 0.060493 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:33:39.185276 | finish at 2026-10-17 18:42:44 + [2026-10-15 06:12:47] iteration 1405/ 2384 | consumed samples: 44960 | elapsed time per iteration (ms): 222875.9 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.74% | FLOPs estimator: workload-aware | logical tokens/s: 75275.7 | valid tokens/s: 75275.7 | physical tokens/s: 76843.0 | loss tokens/s: 62266.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.437047E-01 | lm-global-supervised-tokens: 1.387766E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.760777E-02 | lm-token-buffer-mib: 2.365432E+02 | global_load_balancing_loss: 9.996295E-01 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11172.089 | params rms: 0.060493 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:36:35.548144 | finish at 2026-10-17 18:49:23 + [2026-10-15 06:16:28] iteration 1406/ 2384 | consumed samples: 44992 | elapsed time per iteration (ms): 220651.7 | throughput per GPU (TFLOP/s/GPU): 27.0 | MFU 8.66% | FLOPs estimator: workload-aware | logical tokens/s: 76034.7 | valid tokens/s: 76034.7 | physical tokens/s: 76370.1 | loss tokens/s: 55754.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.895839E-01 | lm-global-supervised-tokens: 1.230236E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.682912E-02 | lm-token-buffer-mib: 2.324194E+02 | global_load_balancing_loss: 9.988178E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11172.055 | params rms: 0.060493 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:56:37.374278 | finish at 2026-10-17 18:13:05 + [2026-10-15 06:20:11] iteration 1407/ 2384 | consumed samples: 45024 | elapsed time per iteration (ms): 222783.9 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 75307.1 | valid tokens/s: 75307.1 | physical tokens/s: 75996.9 | loss tokens/s: 55089.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.220562E-01 | lm-global-supervised-tokens: 1.227297E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.583567E-02 | lm-token-buffer-mib: 2.347871E+02 | global_load_balancing_loss: 1.000301E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11172.018 | params rms: 0.060493 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:27:39.868691 | finish at 2026-10-17 18:47:51 + [2026-10-15 06:23:57] iteration 1408/ 2384 | consumed samples: 45056 | elapsed time per iteration (ms): 226601.8 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 74038.2 | valid tokens/s: 74038.2 | physical tokens/s: 75709.8 | loss tokens/s: 57170.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.176404E-01 | lm-global-supervised-tokens: 1.295497E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.913245E-02 | lm-token-buffer-mib: 2.392335E+02 | global_load_balancing_loss: 1.000359E+00 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11171.981 | params rms: 0.060493 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:26:03.343037 | finish at 2026-10-17 19:50:01 + [2026-10-15 06:27:40] iteration 1409/ 2384 | consumed samples: 45088 | elapsed time per iteration (ms): 223113.1 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 75195.9 | valid tokens/s: 75195.9 | physical tokens/s: 76100.7 | loss tokens/s: 55400.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.575749E-01 | lm-global-supervised-tokens: 1.236057E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.228997E-02 | lm-token-buffer-mib: 2.341277E+02 | global_load_balancing_loss: 1.001781E+00 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11171.946 | params rms: 0.060492 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:25:35.293007 | finish at 2026-10-17 18:53:16 + [2026-10-15 06:31:46] iteration 1410/ 2384 | consumed samples: 45120 | elapsed time per iteration (ms): 245559.8 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 68322.1 | valid tokens/s: 68322.1 | physical tokens/s: 68994.0 | loss tokens/s: 57657.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.203190E-01 | lm-global-supervised-tokens: 1.415835E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.877526E-02 | lm-token-buffer-mib: 2.334127E+02 | global_load_balancing_loss: 1.000861E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11171.911 | params rms: 0.060492 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:26:15.288305 | finish at 2026-10-18 00:58:01 + [2026-10-15 06:35:28] iteration 1411/ 2384 | consumed samples: 45152 | elapsed time per iteration (ms): 221933.8 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.64% | FLOPs estimator: workload-aware | logical tokens/s: 75595.4 | valid tokens/s: 75595.4 | physical tokens/s: 75864.9 | loss tokens/s: 57224.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.723897E-01 | lm-global-supervised-tokens: 1.270009E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.459318E-02 | lm-token-buffer-mib: 2.322064E+02 | global_load_balancing_loss: 1.000565E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11171.875 | params rms: 0.060492 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:59:01.550498 | finish at 2026-10-17 18:34:29 + [2026-10-15 06:39:12] iteration 1412/ 2384 | consumed samples: 45184 | elapsed time per iteration (ms): 224561.3 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 74710.9 | valid tokens/s: 74710.9 | physical tokens/s: 75840.8 | loss tokens/s: 60341.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.179404E-01 | lm-global-supervised-tokens: 1.355024E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.934748E-02 | lm-token-buffer-mib: 2.341074E+02 | global_load_balancing_loss: 1.000597E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11171.840 | params rms: 0.060492 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:37:53.538912 | finish at 2026-10-17 19:17:06 + [2026-10-15 06:42:52] iteration 1413/ 2384 | consumed samples: 45216 | elapsed time per iteration (ms): 219524.0 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 76425.3 | valid tokens/s: 76425.3 | physical tokens/s: 77223.1 | loss tokens/s: 57770.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.385349E-01 | lm-global-supervised-tokens: 1.268194E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.396981E-02 | lm-token-buffer-mib: 2.329583E+02 | global_load_balancing_loss: 9.998805E-01 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11171.803 | params rms: 0.060492 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:12:37.842361 | finish at 2026-10-17 17:55:30 + [2026-10-15 06:46:38] iteration 1414/ 2384 | consumed samples: 45248 | elapsed time per iteration (ms): 225694.6 | throughput per GPU (TFLOP/s/GPU): 25.5 | MFU 8.19% | FLOPs estimator: workload-aware | logical tokens/s: 74335.9 | valid tokens/s: 74335.9 | physical tokens/s: 74662.0 | loss tokens/s: 56197.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.424463E-01 | lm-global-supervised-tokens: 1.268356E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.514129E-02 | lm-token-buffer-mib: 2.330014E+02 | global_load_balancing_loss: 1.000254E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11171.765 | params rms: 0.060491 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:48:43.734350 | finish at 2026-10-17 19:35:21 + [2026-10-15 06:50:22] iteration 1415/ 2384 | consumed samples: 45280 | elapsed time per iteration (ms): 224523.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 74723.8 | valid tokens/s: 74723.8 | physical tokens/s: 75160.5 | loss tokens/s: 58406.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.036171E-01 | lm-global-supervised-tokens: 1.311362E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.521606E-02 | lm-token-buffer-mib: 2.335874E+02 | global_load_balancing_loss: 1.000843E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11171.726 | params rms: 0.060491 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:26:02.908522 | finish at 2026-10-17 19:16:25 + [2026-10-15 06:54:05] iteration 1416/ 2384 | consumed samples: 45312 | elapsed time per iteration (ms): 223249.3 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 75150.0 | valid tokens/s: 75150.0 | physical tokens/s: 75669.9 | loss tokens/s: 57325.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.159119E-01 | lm-global-supervised-tokens: 1.279780E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.178737E-02 | lm-token-buffer-mib: 2.339692E+02 | global_load_balancing_loss: 1.000187E+00 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11171.689 | params rms: 0.060491 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:01:45.301170 | finish at 2026-10-17 18:55:51 + [2026-10-15 06:57:45] iteration 1417/ 2384 | consumed samples: 45344 | elapsed time per iteration (ms): 219341.5 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 76488.8 | valid tokens/s: 76488.8 | physical tokens/s: 76910.9 | loss tokens/s: 53380.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.412692E-01 | lm-global-supervised-tokens: 1.170858E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.664730E-02 | lm-token-buffer-mib: 2.322073E+02 | global_load_balancing_loss: 9.999983E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11171.652 | params rms: 0.060491 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:55:03.197804 | finish at 2026-10-17 17:52:48 + [2026-10-15 07:01:27] iteration 1418/ 2384 | consumed samples: 45376 | elapsed time per iteration (ms): 222349.4 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 75454.1 | valid tokens/s: 75454.1 | physical tokens/s: 75828.8 | loss tokens/s: 59550.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.230268E-01 | lm-global-supervised-tokens: 1.324102E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.670488E-02 | lm-token-buffer-mib: 2.334548E+02 | global_load_balancing_loss: 9.997487E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11171.614 | params rms: 0.060491 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:39:49.493265 | finish at 2026-10-17 18:41:17 + [2026-10-15 07:05:06] iteration 1419/ 2384 | consumed samples: 45408 | elapsed time per iteration (ms): 219206.2 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 76536.1 | valid tokens/s: 76536.1 | physical tokens/s: 77379.1 | loss tokens/s: 57947.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.967037E-01 | lm-global-supervised-tokens: 1.270240E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.589029E-02 | lm-token-buffer-mib: 2.326736E+02 | global_load_balancing_loss: 1.000644E+00 | loss scale: 1.0 | grad norm: 0.275 | params norm: 11171.577 | params rms: 0.060490 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:45:34.019930 | finish at 2026-10-17 17:50:40 + [2026-10-15 07:09:10] iteration 1420/ 2384 | consumed samples: 45440 | elapsed time per iteration (ms): 243228.0 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 68977.1 | valid tokens/s: 68977.1 | physical tokens/s: 69151.0 | loss tokens/s: 53482.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.154421E-01 | lm-global-supervised-tokens: 1.300838E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.340451E-02 | lm-token-buffer-mib: 2.328017E+02 | global_load_balancing_loss: 1.000590E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11171.535 | params rms: 0.060490 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:07:51.745502 | finish at 2026-10-18 00:17:01 + [2026-10-15 07:12:53] iteration 1421/ 2384 | consumed samples: 45472 | elapsed time per iteration (ms): 223945.0 | throughput per GPU (TFLOP/s/GPU): 18.5 | MFU 5.94% | FLOPs estimator: workload-aware | logical tokens/s: 74916.5 | valid tokens/s: 74916.5 | physical tokens/s: 75420.2 | loss tokens/s: 53257.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.142800E-01 | lm-global-supervised-tokens: 1.192676E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.708045E-02 | lm-token-buffer-mib: 2.338353E+02 | global_load_balancing_loss: 1.000236E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11171.496 | params rms: 0.060490 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:54:19.012665 | finish at 2026-10-17 19:07:12 + [2026-10-15 07:16:33] iteration 1422/ 2384 | consumed samples: 45504 | elapsed time per iteration (ms): 219634.5 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 76386.9 | valid tokens/s: 76386.9 | physical tokens/s: 76812.4 | loss tokens/s: 58975.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.608705E-01 | lm-global-supervised-tokens: 1.295312E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.286944E-02 | lm-token-buffer-mib: 2.329200E+02 | global_load_balancing_loss: 1.001948E+00 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11171.460 | params rms: 0.060490 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:41:28.413338 | finish at 2026-10-17 17:58:02 + [2026-10-15 07:20:13] iteration 1423/ 2384 | consumed samples: 45536 | elapsed time per iteration (ms): 220004.6 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.61% | FLOPs estimator: workload-aware | logical tokens/s: 76258.4 | valid tokens/s: 76258.4 | physical tokens/s: 76513.0 | loss tokens/s: 55471.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.617047E-01 | lm-global-supervised-tokens: 1.220398E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.493610E-02 | lm-token-buffer-mib: 2.332171E+02 | global_load_balancing_loss: 9.992663E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11171.424 | params rms: 0.060490 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:43:44.377801 | finish at 2026-10-17 18:03:58 + [2026-10-15 07:23:55] iteration 1424/ 2384 | consumed samples: 45568 | elapsed time per iteration (ms): 221793.5 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75643.4 | valid tokens/s: 75643.3 | physical tokens/s: 76058.3 | loss tokens/s: 53513.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.479492E-01 | lm-global-supervised-tokens: 1.186888E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.909878E-02 | lm-token-buffer-mib: 2.329945E+02 | global_load_balancing_loss: 1.000448E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11171.386 | params rms: 0.060489 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:08:41.766129 | finish at 2026-10-17 18:32:37 + [2026-10-15 07:27:39] iteration 1425/ 2384 | consumed samples: 45600 | elapsed time per iteration (ms): 224382.4 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 74770.6 | valid tokens/s: 74770.6 | physical tokens/s: 75257.8 | loss tokens/s: 59370.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.954720E-01 | lm-global-supervised-tokens: 1.332162E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.230757E-02 | lm-token-buffer-mib: 2.339652E+02 | global_load_balancing_loss: 9.975259E-01 | loss scale: 1.0 | grad norm: 0.322 | params norm: 11171.353 | params rms: 0.060489 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:46:22.703343 | finish at 2026-10-17 19:14:02 + [2026-10-15 07:31:20] iteration 1426/ 2384 | consumed samples: 45632 | elapsed time per iteration (ms): 220857.5 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 75963.9 | valid tokens/s: 75963.9 | physical tokens/s: 76646.4 | loss tokens/s: 59446.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.453922E-01 | lm-global-supervised-tokens: 1.312926E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.404922E-02 | lm-token-buffer-mib: 2.334970E+02 | global_load_balancing_loss: 9.998701E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11171.318 | params rms: 0.060489 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:46:21.464517 | finish at 2026-10-17 18:17:42 + [2026-10-15 07:35:00] iteration 1427/ 2384 | consumed samples: 45664 | elapsed time per iteration (ms): 219955.5 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 76275.3 | valid tokens/s: 76275.3 | physical tokens/s: 76875.8 | loss tokens/s: 56036.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.096875E-01 | lm-global-supervised-tokens: 1.232560E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.457852E-02 | lm-token-buffer-mib: 2.338334E+02 | global_load_balancing_loss: 1.001202E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11171.282 | params rms: 0.060489 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:28:17.460395 | finish at 2026-10-17 18:03:18 + [2026-10-15 07:38:46] iteration 1428/ 2384 | consumed samples: 45696 | elapsed time per iteration (ms): 225491.1 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.16% | FLOPs estimator: workload-aware | logical tokens/s: 74402.9 | valid tokens/s: 74402.8 | physical tokens/s: 76010.6 | loss tokens/s: 57184.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.232827E-01 | lm-global-supervised-tokens: 1.289449E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.389642E-02 | lm-token-buffer-mib: 2.384813E+02 | global_load_balancing_loss: 1.001051E+00 | loss scale: 1.0 | grad norm: 0.180 | params norm: 11171.244 | params rms: 0.060489 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:52:49.535660 | finish at 2026-10-17 19:31:35 + [2026-10-15 07:42:32] iteration 1429/ 2384 | consumed samples: 45728 | elapsed time per iteration (ms): 226653.8 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.17% | FLOPs estimator: workload-aware | logical tokens/s: 74021.3 | valid tokens/s: 74021.3 | physical tokens/s: 74430.5 | loss tokens/s: 56613.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.652009E-01 | lm-global-supervised-tokens: 1.283163E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.995861E-02 | lm-token-buffer-mib: 2.332971E+02 | global_load_balancing_loss: 1.000824E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11171.204 | params rms: 0.060488 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:07:34.403601 | finish at 2026-10-17 19:50:07 + [2026-10-15 07:46:37] iteration 1430/ 2384 | consumed samples: 45760 | elapsed time per iteration (ms): 244670.2 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 68570.5 | valid tokens/s: 68570.5 | physical tokens/s: 69021.4 | loss tokens/s: 51341.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.713993E-01 | lm-global-supervised-tokens: 1.256161E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.179330E-02 | lm-token-buffer-mib: 2.325956E+02 | global_load_balancing_loss: 1.002154E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11171.164 | params rms: 0.060488 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:50:15.393877 | finish at 2026-10-18 00:36:52 + [2026-10-15 07:50:21] iteration 1431/ 2384 | consumed samples: 45792 | elapsed time per iteration (ms): 223956.1 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 74912.9 | valid tokens/s: 74912.9 | physical tokens/s: 75329.9 | loss tokens/s: 58025.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.846085E-01 | lm-global-supervised-tokens: 1.299518E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.836875E-02 | lm-token-buffer-mib: 2.338191E+02 | global_load_balancing_loss: 1.001536E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11171.124 | params rms: 0.060488 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:17:10.130569 | finish at 2026-10-17 19:07:31 + [2026-10-15 07:54:02] iteration 1432/ 2384 | consumed samples: 45824 | elapsed time per iteration (ms): 220956.3 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 75929.8 | valid tokens/s: 75929.8 | physical tokens/s: 76281.1 | loss tokens/s: 53614.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.213872E-01 | lm-global-supervised-tokens: 1.184656E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.605138E-02 | lm-token-buffer-mib: 2.331476E+02 | global_load_balancing_loss: 1.001498E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11171.086 | params rms: 0.060488 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:25:50.428488 | finish at 2026-10-17 18:19:52 + [2026-10-15 07:57:43] iteration 1433/ 2384 | consumed samples: 45856 | elapsed time per iteration (ms): 221105.5 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 75878.5 | valid tokens/s: 75878.4 | physical tokens/s: 76475.9 | loss tokens/s: 53260.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.370202E-01 | lm-global-supervised-tokens: 1.177626E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.695765E-02 | lm-token-buffer-mib: 2.340056E+02 | global_load_balancing_loss: 9.984616E-01 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11171.048 | params rms: 0.060488 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:24:31.344768 | finish at 2026-10-17 18:22:14 + [2026-10-15 08:01:23] iteration 1434/ 2384 | consumed samples: 45888 | elapsed time per iteration (ms): 220043.2 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 76245.0 | valid tokens/s: 76245.0 | physical tokens/s: 77153.4 | loss tokens/s: 51301.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.059245E-01 | lm-global-supervised-tokens: 1.128859E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.935568E-02 | lm-token-buffer-mib: 2.339932E+02 | global_load_balancing_loss: 9.988056E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11171.010 | params rms: 0.060487 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:04:01.067874 | finish at 2026-10-17 18:05:24 + [2026-10-15 08:05:04] iteration 1435/ 2384 | consumed samples: 45920 | elapsed time per iteration (ms): 220915.5 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 75944.0 | valid tokens/s: 75944.0 | physical tokens/s: 76865.0 | loss tokens/s: 59384.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.229426E-01 | lm-global-supervised-tokens: 1.311898E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.667013E-02 | lm-token-buffer-mib: 2.354863E+02 | global_load_balancing_loss: 9.981475E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11170.973 | params rms: 0.060487 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:14:08.852871 | finish at 2026-10-17 18:19:13 + [2026-10-15 08:08:51] iteration 1436/ 2384 | consumed samples: 45952 | elapsed time per iteration (ms): 226844.3 | throughput per GPU (TFLOP/s/GPU): 26.5 | MFU 8.48% | FLOPs estimator: workload-aware | logical tokens/s: 73959.0 | valid tokens/s: 73959.0 | physical tokens/s: 74673.2 | loss tokens/s: 56036.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.014760E-01 | lm-global-supervised-tokens: 1.271166E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.268311E-02 | lm-token-buffer-mib: 2.356450E+02 | global_load_balancing_loss: 9.991318E-01 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11170.935 | params rms: 0.060487 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:44:08.402533 | finish at 2026-10-17 19:52:59 + [2026-10-15 08:12:33] iteration 1437/ 2384 | consumed samples: 45984 | elapsed time per iteration (ms): 221786.7 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 75645.7 | valid tokens/s: 75645.7 | physical tokens/s: 76180.7 | loss tokens/s: 56229.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.954725E-01 | lm-global-supervised-tokens: 1.247092E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.459732E-02 | lm-token-buffer-mib: 2.340122E+02 | global_load_balancing_loss: 9.998967E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11170.894 | params rms: 0.060487 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:20:31.964269 | finish at 2026-10-17 18:33:05 + [2026-10-15 08:16:18] iteration 1438/ 2384 | consumed samples: 46016 | elapsed time per iteration (ms): 225179.2 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 74505.9 | valid tokens/s: 74505.8 | physical tokens/s: 75612.8 | loss tokens/s: 58015.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.975650E-01 | lm-global-supervised-tokens: 1.306393E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.396286E-02 | lm-token-buffer-mib: 2.370179E+02 | global_load_balancing_loss: 1.001556E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11170.850 | params rms: 0.060487 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:10:19.512988 | finish at 2026-10-17 19:26:37 + [2026-10-15 08:20:01] iteration 1439/ 2384 | consumed samples: 46048 | elapsed time per iteration (ms): 223538.7 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 75052.8 | valid tokens/s: 75052.8 | physical tokens/s: 75974.7 | loss tokens/s: 60832.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.025442E-01 | lm-global-supervised-tokens: 1.359843E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.637525E-02 | lm-token-buffer-mib: 2.334921E+02 | global_load_balancing_loss: 1.001342E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11170.809 | params rms: 0.060486 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:40:44.075203 | finish at 2026-10-17 19:00:45 + [2026-10-15 08:24:05] iteration 1440/ 2384 | consumed samples: 46080 | elapsed time per iteration (ms): 243902.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 68786.6 | valid tokens/s: 68786.6 | physical tokens/s: 69294.2 | loss tokens/s: 52369.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.399855E-01 | lm-global-supervised-tokens: 1.277300E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.877447E-02 | lm-token-buffer-mib: 2.344464E+02 | global_load_balancing_loss: 1.002984E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11170.771 | params rms: 0.060486 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:57:23.572578 | finish at 2026-10-18 00:21:29 +times across ranks (ms): + save-checkpoint: + rank 0: 24919.37 + rank 1: 24919.33 + rank 2: 24919.38 + rank 3: 24919.37 + rank 4: 24919.34 + rank 5: 24919.36 + rank 6: 24919.36 + rank 7: 24919.35 + rank 8: 24918.83 + rank 9: 24918.84 + rank 10: 24918.86 + rank 11: 24918.84 + rank 12: 24918.85 + rank 13: 24918.83 + rank 14: 24918.85 + rank 15: 24918.83 + rank 16: 24919.10 + rank 17: 24919.10 + rank 18: 24919.10 + rank 19: 24919.09 + rank 20: 24919.06 + rank 21: 24919.06 + rank 22: 24918.97 + rank 23: 24919.04 + rank 24: 24919.06 + rank 25: 24919.02 + rank 26: 24919.13 + rank 27: 24919.05 + rank 28: 24918.97 + rank 29: 24919.08 + rank 30: 24918.98 + rank 31: 24919.01 + rank 32: 24918.29 + rank 33: 24918.29 + rank 34: 24918.43 + rank 35: 24918.47 + rank 36: 24918.35 + rank 37: 24918.40 + rank 38: 24918.35 + rank 39: 24918.34 + rank 40: 24918.71 + rank 41: 24918.72 + rank 42: 24918.72 + rank 43: 24918.74 + rank 44: 24918.75 + rank 45: 24918.73 + rank 46: 24918.72 + rank 47: 24918.72 + rank 48: 24918.78 + rank 49: 24918.84 + rank 50: 24918.83 + rank 51: 24918.77 + rank 52: 24918.79 + rank 53: 24919.02 + rank 54: 24918.76 + rank 55: 24918.84 + rank 56: 24920.27 + rank 57: 24920.30 + rank 58: 24920.32 + rank 59: 24920.16 + rank 60: 24920.30 + rank 61: 24920.31 + rank 62: 24920.26 + rank 63: 24920.30 + [2026-10-15 08:28:23] iteration 1441/ 2384 | consumed samples: 46112 | elapsed time per iteration (ms): 232763.9 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 72078.2 | valid tokens/s: 72078.2 | physical tokens/s: 72539.3 | loss tokens/s: 54981.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.622444E-01 | lm-global-supervised-tokens: 1.279767E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.146071E-01 | lm-token-buffer-mib: 2.342235E+02 | global_load_balancing_loss: 1.002879E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11170.735 | params rms: 0.060486 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:58:16.393937 | finish at 2026-10-17 21:26:39 + [2026-10-15 08:32:14] iteration 1442/ 2384 | consumed samples: 46144 | elapsed time per iteration (ms): 231416.9 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 72497.7 | valid tokens/s: 72497.7 | physical tokens/s: 73399.9 | loss tokens/s: 58733.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.014608E-01 | lm-global-supervised-tokens: 1.359193E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.295757E-02 | lm-token-buffer-mib: 2.352914E+02 | global_load_balancing_loss: 1.000380E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11170.700 | params rms: 0.060486 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:33:14.686036 | finish at 2026-10-17 21:05:29 + [2026-10-15 08:35:56] iteration 1443/ 2384 | consumed samples: 46176 | elapsed time per iteration (ms): 221863.8 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.48% | FLOPs estimator: workload-aware | logical tokens/s: 75619.4 | valid tokens/s: 75619.4 | physical tokens/s: 76509.6 | loss tokens/s: 58189.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.943210E-01 | lm-global-supervised-tokens: 1.291007E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.300075E-02 | lm-token-buffer-mib: 2.346780E+02 | global_load_balancing_loss: 9.985027E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11170.669 | params rms: 0.060486 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:59:33.840109 | finish at 2026-10-17 18:35:30 + [2026-10-15 08:39:39] iteration 1444/ 2384 | consumed samples: 46208 | elapsed time per iteration (ms): 222267.7 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 75482.0 | valid tokens/s: 75482.0 | physical tokens/s: 76086.7 | loss tokens/s: 58432.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.945305E-01 | lm-global-supervised-tokens: 1.298761E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.173872E-02 | lm-token-buffer-mib: 2.340530E+02 | global_load_balancing_loss: 9.978081E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11170.636 | params rms: 0.060485 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:02:11.592016 | finish at 2026-10-17 18:41:50 + [2026-10-15 08:43:19] iteration 1445/ 2384 | consumed samples: 46240 | elapsed time per iteration (ms): 220861.5 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 75962.5 | valid tokens/s: 75962.5 | physical tokens/s: 77055.9 | loss tokens/s: 56052.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.930296E-01 | lm-global-supervised-tokens: 1.237992E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.883465E-02 | lm-token-buffer-mib: 2.348162E+02 | global_load_balancing_loss: 9.978796E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11170.602 | params rms: 0.060485 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:36:28.982105 | finish at 2026-10-17 18:19:48 + [2026-10-15 08:47:04] iteration 1446/ 2384 | consumed samples: 46272 | elapsed time per iteration (ms): 224219.8 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 74824.8 | valid tokens/s: 74824.8 | physical tokens/s: 75635.8 | loss tokens/s: 57575.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.407272E-01 | lm-global-supervised-tokens: 1.290956E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.361590E-02 | lm-token-buffer-mib: 2.353054E+02 | global_load_balancing_loss: 9.982684E-01 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11170.568 | params rms: 0.060485 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:25:18.132812 | finish at 2026-10-17 19:12:22 + [2026-10-15 08:50:44] iteration 1447/ 2384 | consumed samples: 46304 | elapsed time per iteration (ms): 220364.4 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 76133.9 | valid tokens/s: 76133.9 | physical tokens/s: 77689.2 | loss tokens/s: 58059.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.577459E-01 | lm-global-supervised-tokens: 1.279426E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.482985E-02 | lm-token-buffer-mib: 2.340254E+02 | global_load_balancing_loss: 1.000747E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11170.534 | params rms: 0.060485 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:21:21.452992 | finish at 2026-10-17 18:12:05 + [2026-10-15 08:54:27] iteration 1448/ 2384 | consumed samples: 46336 | elapsed time per iteration (ms): 223383.9 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75104.8 | valid tokens/s: 75104.8 | physical tokens/s: 75862.1 | loss tokens/s: 57021.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.972372E-01 | lm-global-supervised-tokens: 1.273767E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.614167E-02 | lm-token-buffer-mib: 2.354865E+02 | global_load_balancing_loss: 1.002102E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11170.502 | params rms: 0.060485 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:04:47.345060 | finish at 2026-10-17 18:59:15 + [2026-10-15 08:58:09] iteration 1449/ 2384 | consumed samples: 46368 | elapsed time per iteration (ms): 221124.4 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.36% | FLOPs estimator: workload-aware | logical tokens/s: 75871.9 | valid tokens/s: 75871.9 | physical tokens/s: 76859.5 | loss tokens/s: 60671.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.774756E-01 | lm-global-supervised-tokens: 1.341585E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.295573E-02 | lm-token-buffer-mib: 2.358106E+02 | global_load_balancing_loss: 1.002942E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11170.474 | params rms: 0.060484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:25:51.309226 | finish at 2026-10-17 18:24:00 + [2026-10-15 09:02:13] iteration 1450/ 2384 | consumed samples: 46400 | elapsed time per iteration (ms): 244009.6 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 68756.3 | valid tokens/s: 68756.3 | physical tokens/s: 69385.3 | loss tokens/s: 50257.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.950789E-01 | lm-global-supervised-tokens: 1.226326E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.578554E-02 | lm-token-buffer-mib: 2.335272E+02 | global_load_balancing_loss: 1.002553E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11170.446 | params rms: 0.060484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:18:24.962098 | finish at 2026-10-18 00:20:37 + [2026-10-15 09:05:56] iteration 1451/ 2384 | consumed samples: 46432 | elapsed time per iteration (ms): 223385.4 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 5.99% | FLOPs estimator: workload-aware | logical tokens/s: 75104.3 | valid tokens/s: 75104.3 | physical tokens/s: 75954.1 | loss tokens/s: 57999.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.455409E-01 | lm-global-supervised-tokens: 1.295623E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.374863E-02 | lm-token-buffer-mib: 2.357766E+02 | global_load_balancing_loss: 1.000619E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11170.414 | params rms: 0.060484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:53:38.546888 | finish at 2026-10-17 18:59:34 + [2026-10-15 09:09:38] iteration 1452/ 2384 | consumed samples: 46464 | elapsed time per iteration (ms): 221782.9 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 75646.8 | valid tokens/s: 75646.8 | physical tokens/s: 75955.2 | loss tokens/s: 58651.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.786160E-01 | lm-global-supervised-tokens: 1.300787E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.151722E-02 | lm-token-buffer-mib: 2.326694E+02 | global_load_balancing_loss: 9.992835E-01 | loss scale: 1.0 | grad norm: 0.518 | params norm: 11170.377 | params rms: 0.060484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:25:01.661111 | finish at 2026-10-17 18:34:39 + [2026-10-15 09:13:24] iteration 1453/ 2384 | consumed samples: 46496 | elapsed time per iteration (ms): 226769.8 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 73983.3 | valid tokens/s: 73983.3 | physical tokens/s: 75685.9 | loss tokens/s: 60054.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.728504E-01 | lm-global-supervised-tokens: 1.361861E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.441850E-02 | lm-token-buffer-mib: 2.400426E+02 | global_load_balancing_loss: 9.994856E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11170.326 | params rms: 0.060484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:38:42.641134 | finish at 2026-10-17 19:52:07 + [2026-10-15 09:17:09] iteration 1454/ 2384 | consumed samples: 46528 | elapsed time per iteration (ms): 224436.5 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 74752.5 | valid tokens/s: 74752.5 | physical tokens/s: 75276.2 | loss tokens/s: 55438.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.310674E-01 | lm-global-supervised-tokens: 1.244245E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.395857E-02 | lm-token-buffer-mib: 2.339264E+02 | global_load_balancing_loss: 1.000050E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11170.287 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:58:45.965910 | finish at 2026-10-17 19:15:55 + [2026-10-15 09:20:54] iteration 1455/ 2384 | consumed samples: 46560 | elapsed time per iteration (ms): 225339.9 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 74452.8 | valid tokens/s: 74452.8 | physical tokens/s: 75296.7 | loss tokens/s: 56604.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.877314E-01 | lm-global-supervised-tokens: 1.275513E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.417360E-02 | lm-token-buffer-mib: 2.346468E+02 | global_load_balancing_loss: 1.000398E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11170.256 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:09:00.787714 | finish at 2026-10-17 19:29:55 + [2026-10-15 09:24:38] iteration 1456/ 2384 | consumed samples: 46592 | elapsed time per iteration (ms): 223659.8 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 75012.0 | valid tokens/s: 75012.0 | physical tokens/s: 75535.3 | loss tokens/s: 58096.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.759669E-01 | lm-global-supervised-tokens: 1.299386E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.246773E-02 | lm-token-buffer-mib: 2.345433E+02 | global_load_balancing_loss: 9.998999E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11170.233 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:39:16.333168 | finish at 2026-10-17 19:03:54 + [2026-10-15 09:28:20] iteration 1457/ 2384 | consumed samples: 46624 | elapsed time per iteration (ms): 222542.0 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 75389.0 | valid tokens/s: 75389.0 | physical tokens/s: 76473.2 | loss tokens/s: 55270.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.395988E-01 | lm-global-supervised-tokens: 1.229993E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.804596E-02 | lm-token-buffer-mib: 2.358770E+02 | global_load_balancing_loss: 1.000613E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11170.214 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:18:16.407972 | finish at 2026-10-17 18:46:37 + [2026-10-15 09:32:02] iteration 1458/ 2384 | consumed samples: 46656 | elapsed time per iteration (ms): 221890.7 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.11% | FLOPs estimator: workload-aware | logical tokens/s: 75610.3 | valid tokens/s: 75610.3 | physical tokens/s: 76402.3 | loss tokens/s: 59720.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.477200E-01 | lm-global-supervised-tokens: 1.325152E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.271761E-02 | lm-token-buffer-mib: 2.342970E+02 | global_load_balancing_loss: 1.000558E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11170.199 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:04:30.744581 | finish at 2026-10-17 18:36:33 + [2026-10-15 09:35:48] iteration 1459/ 2384 | consumed samples: 46688 | elapsed time per iteration (ms): 225273.7 | throughput per GPU (TFLOP/s/GPU): 27.4 | MFU 8.80% | FLOPs estimator: workload-aware | logical tokens/s: 74474.7 | valid tokens/s: 74474.7 | physical tokens/s: 74656.6 | loss tokens/s: 57490.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.169563E-01 | lm-global-supervised-tokens: 1.295098E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.486490E-02 | lm-token-buffer-mib: 2.326639E+02 | global_load_balancing_loss: 1.000614E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11170.182 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:52:58.179777 | finish at 2026-10-17 19:28:46 + [2026-10-15 09:39:50] iteration 1460/ 2384 | consumed samples: 46720 | elapsed time per iteration (ms): 241903.5 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.16% | FLOPs estimator: workload-aware | logical tokens/s: 69355.0 | valid tokens/s: 69355.0 | physical tokens/s: 70060.6 | loss tokens/s: 51168.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.891386E-01 | lm-global-supervised-tokens: 1.237779E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.675444E-02 | lm-token-buffer-mib: 2.335969E+02 | global_load_balancing_loss: 1.000565E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11170.162 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:05:18.827685 | finish at 2026-10-17 23:45:08 + [2026-10-15 09:43:32] iteration 1461/ 2384 | consumed samples: 46752 | elapsed time per iteration (ms): 222711.2 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 75331.5 | valid tokens/s: 75331.5 | physical tokens/s: 76550.4 | loss tokens/s: 60342.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.202555E-01 | lm-global-supervised-tokens: 1.343896E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.486220E-02 | lm-token-buffer-mib: 2.339131E+02 | global_load_balancing_loss: 1.000502E+00 | loss scale: 1.0 | grad norm: 0.346 | params norm: 11170.144 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:06:02.452343 | finish at 2026-10-17 18:49:35 + [2026-10-15 09:47:14] iteration 1462/ 2384 | consumed samples: 46784 | elapsed time per iteration (ms): 222225.6 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 75495.9 | valid tokens/s: 75495.9 | physical tokens/s: 75818.9 | loss tokens/s: 54794.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.179566E-01 | lm-global-supervised-tokens: 1.217672E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.597654E-02 | lm-token-buffer-mib: 2.326160E+02 | global_load_balancing_loss: 9.995657E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11170.130 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:54:52.035298 | finish at 2026-10-17 18:42:06 + [2026-10-15 09:50:57] iteration 1463/ 2384 | consumed samples: 46816 | elapsed time per iteration (ms): 222092.9 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 75541.4 | valid tokens/s: 75541.4 | physical tokens/s: 76361.8 | loss tokens/s: 55244.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.750063E-01 | lm-global-supervised-tokens: 1.226945E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.119009E-02 | lm-token-buffer-mib: 2.338407E+02 | global_load_balancing_loss: 1.000335E+00 | loss scale: 1.0 | grad norm: 0.852 | params norm: 11170.118 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:49:07.565107 | finish at 2026-10-17 18:40:04 + [2026-10-15 09:54:39] iteration 1464/ 2384 | consumed samples: 46848 | elapsed time per iteration (ms): 222238.8 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 75491.7 | valid tokens/s: 75491.7 | physical tokens/s: 75744.7 | loss tokens/s: 56057.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.690268E-01 | lm-global-supervised-tokens: 1.245818E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.294123E-02 | lm-token-buffer-mib: 2.323789E+02 | global_load_balancing_loss: 1.000842E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11170.103 | params rms: 0.060482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:47:39.672794 | finish at 2026-10-17 18:42:18 + [2026-10-15 09:58:19] iteration 1465/ 2384 | consumed samples: 46880 | elapsed time per iteration (ms): 220374.2 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.26% | FLOPs estimator: workload-aware | logical tokens/s: 76130.5 | valid tokens/s: 76130.5 | physical tokens/s: 76762.8 | loss tokens/s: 59046.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.234158E-01 | lm-global-supervised-tokens: 1.301225E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.515376E-02 | lm-token-buffer-mib: 2.346332E+02 | global_load_balancing_loss: 9.996991E-01 | loss scale: 1.0 | grad norm: 0.423 | params norm: 11170.086 | params rms: 0.060482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:15:23.864919 | finish at 2026-10-17 18:13:43 + [2026-10-15 10:02:04] iteration 1466/ 2384 | consumed samples: 46912 | elapsed time per iteration (ms): 224746.9 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 74649.2 | valid tokens/s: 74649.2 | physical tokens/s: 75496.8 | loss tokens/s: 59105.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.960603E-01 | lm-global-supervised-tokens: 1.328370E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.426167E-02 | lm-token-buffer-mib: 2.359827E+02 | global_load_balancing_loss: 9.998307E-01 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11170.068 | params rms: 0.060482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:18:37.632388 | finish at 2026-10-17 19:20:42 + [2026-10-15 10:05:50] iteration 1467/ 2384 | consumed samples: 46944 | elapsed time per iteration (ms): 225612.6 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 74362.9 | valid tokens/s: 74362.9 | physical tokens/s: 75130.6 | loss tokens/s: 57285.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.338768E-01 | lm-global-supervised-tokens: 1.292424E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.259514E-02 | lm-token-buffer-mib: 2.344998E+02 | global_load_balancing_loss: 1.000575E+00 | loss scale: 1.0 | grad norm: 0.323 | params norm: 11170.048 | params rms: 0.060482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:28:06.735479 | finish at 2026-10-17 19:33:56 + [2026-10-15 10:09:36] iteration 1468/ 2384 | consumed samples: 46976 | elapsed time per iteration (ms): 226560.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 74051.8 | valid tokens/s: 74051.8 | physical tokens/s: 74823.1 | loss tokens/s: 52567.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.885962E-01 | lm-global-supervised-tokens: 1.190973E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.688417E-02 | lm-token-buffer-mib: 2.365027E+02 | global_load_balancing_loss: 1.000112E+00 | loss scale: 1.0 | grad norm: 0.225 | params norm: 11170.024 | params rms: 0.060482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:38:49.035293 | finish at 2026-10-17 19:48:25 + [2026-10-15 10:13:22] iteration 1469/ 2384 | consumed samples: 47008 | elapsed time per iteration (ms): 225660.6 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 74347.1 | valid tokens/s: 74347.0 | physical tokens/s: 75294.4 | loss tokens/s: 55436.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.048062E-01 | lm-global-supervised-tokens: 1.250983E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.064649E-02 | lm-token-buffer-mib: 2.362825E+02 | global_load_balancing_loss: 1.001217E+00 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11169.997 | params rms: 0.060482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:21:19.488437 | finish at 2026-10-17 19:34:41 + [2026-10-15 10:17:25] iteration 1470/ 2384 | consumed samples: 47040 | elapsed time per iteration (ms): 243364.6 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 5.99% | FLOPs estimator: workload-aware | logical tokens/s: 68938.5 | valid tokens/s: 68938.5 | physical tokens/s: 69365.9 | loss tokens/s: 50834.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.454310E-01 | lm-global-supervised-tokens: 1.237125E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.608647E-02 | lm-token-buffer-mib: 2.330529E+02 | global_load_balancing_loss: 1.001039E+00 | loss scale: 1.0 | grad norm: 0.168 | params norm: 11169.969 | params rms: 0.060482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:47:15.272895 | finish at 2026-10-18 00:04:40 + [2026-10-15 10:21:09] iteration 1471/ 2384 | consumed samples: 47072 | elapsed time per iteration (ms): 223796.4 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 74966.1 | valid tokens/s: 74966.1 | physical tokens/s: 75563.0 | loss tokens/s: 55633.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.817555E-01 | lm-global-supervised-tokens: 1.245055E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.427831E-02 | lm-token-buffer-mib: 2.352782E+02 | global_load_balancing_loss: 1.000954E+00 | loss scale: 1.0 | grad norm: 0.180 | params norm: 11169.935 | params rms: 0.060482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:45:26.140474 | finish at 2026-10-17 19:06:35 + [2026-10-15 10:24:50] iteration 1472/ 2384 | consumed samples: 47104 | elapsed time per iteration (ms): 221469.7 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 75753.9 | valid tokens/s: 75753.9 | physical tokens/s: 76253.4 | loss tokens/s: 58895.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.261379E-01 | lm-global-supervised-tokens: 1.304356E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.282472E-02 | lm-token-buffer-mib: 2.331849E+02 | global_load_balancing_loss: 1.000402E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11169.903 | params rms: 0.060481 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:06:20.328873 | finish at 2026-10-17 18:31:11 + [2026-10-15 10:28:29] iteration 1473/ 2384 | consumed samples: 47136 | elapsed time per iteration (ms): 218910.9 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 76639.3 | valid tokens/s: 76639.3 | physical tokens/s: 77297.0 | loss tokens/s: 53964.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.940129E-01 | lm-global-supervised-tokens: 1.181335E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.416064E-02 | lm-token-buffer-mib: 2.330385E+02 | global_load_balancing_loss: 9.997693E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11169.870 | params rms: 0.060481 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:23:47.839345 | finish at 2026-10-17 17:52:17 + [2026-10-15 10:32:09] iteration 1474/ 2384 | consumed samples: 47168 | elapsed time per iteration (ms): 219801.6 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 76328.8 | valid tokens/s: 76328.8 | physical tokens/s: 76728.1 | loss tokens/s: 56963.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.780036E-01 | lm-global-supervised-tokens: 1.252064E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.662376E-02 | lm-token-buffer-mib: 2.337114E+02 | global_load_balancing_loss: 1.000087E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11169.833 | params rms: 0.060481 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:33:39.456198 | finish at 2026-10-17 18:05:49 + [2026-10-15 10:35:52] iteration 1475/ 2384 | consumed samples: 47200 | elapsed time per iteration (ms): 222634.8 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75357.4 | valid tokens/s: 75357.4 | physical tokens/s: 76207.6 | loss tokens/s: 60159.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.677834E-01 | lm-global-supervised-tokens: 1.339356E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.835076E-02 | lm-token-buffer-mib: 2.361082E+02 | global_load_balancing_loss: 1.001536E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11169.798 | params rms: 0.060481 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:12:55.030777 | finish at 2026-10-17 18:48:47 + [2026-10-15 10:39:35] iteration 1476/ 2384 | consumed samples: 47232 | elapsed time per iteration (ms): 223010.8 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 75230.5 | valid tokens/s: 75230.5 | physical tokens/s: 76571.0 | loss tokens/s: 58062.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.456106E-01 | lm-global-supervised-tokens: 1.294849E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.791022E-02 | lm-token-buffer-mib: 2.336841E+02 | global_load_balancing_loss: 9.990932E-01 | loss scale: 1.0 | grad norm: 0.201 | params norm: 11169.763 | params rms: 0.060481 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:14:53.796337 | finish at 2026-10-17 18:54:29 + [2026-10-15 10:43:19] iteration 1477/ 2384 | consumed samples: 47264 | elapsed time per iteration (ms): 223998.0 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 74898.9 | valid tokens/s: 74898.9 | physical tokens/s: 75596.7 | loss tokens/s: 59784.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.237325E-01 | lm-global-supervised-tokens: 1.339168E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.872733E-02 | lm-token-buffer-mib: 2.337637E+02 | global_load_balancing_loss: 1.000180E+00 | loss scale: 1.0 | grad norm: 0.216 | params norm: 11169.726 | params rms: 0.060480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:26:06.208189 | finish at 2026-10-17 19:09:25 + [2026-10-15 10:46:58] iteration 1478/ 2384 | consumed samples: 47296 | elapsed time per iteration (ms): 219694.7 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.10% | FLOPs estimator: workload-aware | logical tokens/s: 76366.0 | valid tokens/s: 76365.9 | physical tokens/s: 77018.9 | loss tokens/s: 57925.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.337266E-01 | lm-global-supervised-tokens: 1.272597E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.395744E-02 | lm-token-buffer-mib: 2.331565E+02 | global_load_balancing_loss: 1.000748E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11169.688 | params rms: 0.060480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:17:23.391722 | finish at 2026-10-17 18:04:22 + [2026-10-15 10:50:40] iteration 1479/ 2384 | consumed samples: 47328 | elapsed time per iteration (ms): 221207.1 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 75843.8 | valid tokens/s: 75843.8 | physical tokens/s: 76704.4 | loss tokens/s: 57283.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.003215E-01 | lm-global-supervised-tokens: 1.267154E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.490263E-02 | lm-token-buffer-mib: 2.341455E+02 | global_load_balancing_loss: 1.001047E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11169.649 | params rms: 0.060480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:36:32.433838 | finish at 2026-10-17 18:27:12 + [2026-10-15 10:54:43] iteration 1480/ 2384 | consumed samples: 47360 | elapsed time per iteration (ms): 243409.7 | throughput per GPU (TFLOP/s/GPU): 16.7 | MFU 5.36% | FLOPs estimator: workload-aware | logical tokens/s: 68925.8 | valid tokens/s: 68925.8 | physical tokens/s: 69719.4 | loss tokens/s: 52777.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.653504E-01 | lm-global-supervised-tokens: 1.284648E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.534371E-02 | lm-token-buffer-mib: 2.354587E+02 | global_load_balancing_loss: 9.998085E-01 | loss scale: 1.0 | grad norm: 0.307 | params norm: 11169.607 | params rms: 0.060480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:07:22.324541 | finish at 2026-10-18 00:02:05 +times across ranks (ms): + save-checkpoint: + rank 0: 23509.03 + rank 1: 23509.01 + rank 2: 23509.03 + rank 3: 23509.07 + rank 4: 23509.00 + rank 5: 23509.05 + rank 6: 23509.01 + rank 7: 23509.01 + rank 8: 23509.90 + rank 9: 23509.93 + rank 10: 23509.91 + rank 11: 23509.89 + rank 12: 23509.95 + rank 13: 23509.90 + rank 14: 23509.90 + rank 15: 23509.90 + rank 16: 23509.32 + rank 17: 23509.30 + rank 18: 23509.30 + rank 19: 23509.28 + rank 20: 23509.32 + rank 21: 23509.30 + rank 22: 23509.33 + rank 23: 23509.32 + rank 24: 23508.75 + rank 25: 23508.81 + rank 26: 23508.78 + rank 27: 23508.79 + rank 28: 23508.73 + rank 29: 23508.77 + rank 30: 23508.73 + rank 31: 23508.75 + rank 32: 23509.14 + rank 33: 23509.18 + rank 34: 23509.12 + rank 35: 23509.16 + rank 36: 23509.11 + rank 37: 23509.15 + rank 38: 23509.11 + rank 39: 23509.11 + rank 40: 23509.34 + rank 41: 23509.37 + rank 42: 23509.35 + rank 43: 23509.40 + rank 44: 23509.35 + rank 45: 23509.33 + rank 46: 23509.33 + rank 47: 23509.33 + rank 48: 23508.91 + rank 49: 23508.93 + rank 50: 23508.93 + rank 51: 23508.91 + rank 52: 23508.92 + rank 53: 23508.93 + rank 54: 23508.90 + rank 55: 23508.91 + rank 56: 23509.03 + rank 57: 23509.02 + rank 58: 23509.02 + rank 59: 23509.03 + rank 60: 23509.06 + rank 61: 23509.03 + rank 62: 23509.01 + rank 63: 23508.99 + [2026-10-15 10:59:02] iteration 1481/ 2384 | consumed samples: 47392 | elapsed time per iteration (ms): 235184.8 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 71336.2 | valid tokens/s: 71336.2 | physical tokens/s: 71971.5 | loss tokens/s: 56090.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.495909E-01 | lm-global-supervised-tokens: 1.319171E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.868185E-01 | lm-token-buffer-mib: 2.343676E+02 | global_load_balancing_loss: 1.000645E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11169.565 | params rms: 0.060480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:59:31.862477 | finish at 2026-10-17 21:58:34 + [2026-10-15 11:02:50] iteration 1482/ 2384 | consumed samples: 47424 | elapsed time per iteration (ms): 227874.8 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 73624.6 | valid tokens/s: 73624.6 | physical tokens/s: 74126.6 | loss tokens/s: 53439.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.379928E-01 | lm-global-supervised-tokens: 1.217754E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.160890E-02 | lm-token-buffer-mib: 2.344956E+02 | global_load_balancing_loss: 1.000757E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11169.520 | params rms: 0.060479 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:05:43.045054 | finish at 2026-10-17 20:08:33 + [2026-10-15 11:06:39] iteration 1483/ 2384 | consumed samples: 47456 | elapsed time per iteration (ms): 228796.5 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 73327.9 | valid tokens/s: 73327.9 | physical tokens/s: 75107.7 | loss tokens/s: 56324.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.238029E-01 | lm-global-supervised-tokens: 1.288693E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.980743E-02 | lm-token-buffer-mib: 2.421311E+02 | global_load_balancing_loss: 9.999491E-01 | loss scale: 1.0 | grad norm: 0.394 | params norm: 11169.474 | params rms: 0.060479 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:15:45.641315 | finish at 2026-10-17 20:22:24 + [2026-10-15 11:10:18] iteration 1484/ 2384 | consumed samples: 47488 | elapsed time per iteration (ms): 219884.4 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 76300.0 | valid tokens/s: 76300.0 | physical tokens/s: 76579.6 | loss tokens/s: 56356.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.844830E-01 | lm-global-supervised-tokens: 1.239197E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.838183E-02 | lm-token-buffer-mib: 2.324361E+02 | global_load_balancing_loss: 9.998768E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11169.429 | params rms: 0.060479 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:58:15.966768 | finish at 2026-10-17 18:08:34 + [2026-10-15 11:14:05] iteration 1485/ 2384 | consumed samples: 47520 | elapsed time per iteration (ms): 226653.4 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.66% | FLOPs estimator: workload-aware | logical tokens/s: 74021.3 | valid tokens/s: 74021.3 | physical tokens/s: 74786.4 | loss tokens/s: 58388.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.968041E-01 | lm-global-supervised-tokens: 1.323404E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.234461E-02 | lm-token-buffer-mib: 2.341949E+02 | global_load_balancing_loss: 1.001330E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11169.383 | params rms: 0.060479 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:36:01.438701 | finish at 2026-10-17 19:50:07 + [2026-10-15 11:17:45] iteration 1486/ 2384 | consumed samples: 47552 | elapsed time per iteration (ms): 220148.2 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.02% | FLOPs estimator: workload-aware | logical tokens/s: 76208.5 | valid tokens/s: 76208.5 | physical tokens/s: 76436.4 | loss tokens/s: 56304.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.770707E-01 | lm-global-supervised-tokens: 1.239535E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.474000E-02 | lm-token-buffer-mib: 2.331679E+02 | global_load_balancing_loss: 9.999237E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11169.336 | params rms: 0.060478 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:54:53.056441 | finish at 2026-10-17 18:12:38 + [2026-10-15 11:21:27] iteration 1487/ 2384 | consumed samples: 47584 | elapsed time per iteration (ms): 221480.7 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75750.0 | valid tokens/s: 75750.0 | physical tokens/s: 76509.3 | loss tokens/s: 57893.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.152225E-01 | lm-global-supervised-tokens: 1.282233E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.201789E-02 | lm-token-buffer-mib: 2.334568E+02 | global_load_balancing_loss: 1.000386E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11169.288 | params rms: 0.060478 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:11:08.224699 | finish at 2026-10-17 18:32:35 + [2026-10-15 11:25:12] iteration 1488/ 2384 | consumed samples: 47616 | elapsed time per iteration (ms): 225391.3 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 74435.8 | valid tokens/s: 74435.8 | physical tokens/s: 75047.0 | loss tokens/s: 56059.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.789199E-01 | lm-global-supervised-tokens: 1.263542E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.431577E-02 | lm-token-buffer-mib: 2.346189E+02 | global_load_balancing_loss: 1.000172E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11169.242 | params rms: 0.060478 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:05:50.624207 | finish at 2026-10-17 19:31:03 + [2026-10-15 11:28:54] iteration 1489/ 2384 | consumed samples: 47648 | elapsed time per iteration (ms): 221591.4 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 75712.2 | valid tokens/s: 75712.2 | physical tokens/s: 76393.2 | loss tokens/s: 51966.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.779920E-01 | lm-global-supervised-tokens: 1.151541E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.911368E-02 | lm-token-buffer-mib: 2.333787E+02 | global_load_balancing_loss: 1.000447E+00 | loss scale: 1.0 | grad norm: 0.183 | params norm: 11169.197 | params rms: 0.060478 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:05:24.280086 | finish at 2026-10-17 18:34:18 + [2026-10-15 11:32:57] iteration 1490/ 2384 | consumed samples: 47680 | elapsed time per iteration (ms): 243565.9 | throughput per GPU (TFLOP/s/GPU): 18.1 | MFU 5.80% | FLOPs estimator: workload-aware | logical tokens/s: 68881.6 | valid tokens/s: 68881.6 | physical tokens/s: 69350.4 | loss tokens/s: 48965.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.300278E-01 | lm-global-supervised-tokens: 1.192626E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.369513E-02 | lm-token-buffer-mib: 2.347561E+02 | global_load_balancing_loss: 9.990367E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11169.147 | params rms: 0.060477 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:29:07.917236 | finish at 2026-10-18 00:02:05 + [2026-10-15 11:36:40] iteration 1491/ 2384 | consumed samples: 47712 | elapsed time per iteration (ms): 222322.9 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.76% | FLOPs estimator: workload-aware | logical tokens/s: 75463.2 | valid tokens/s: 75463.2 | physical tokens/s: 76223.5 | loss tokens/s: 59407.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.469973E-01 | lm-global-supervised-tokens: 1.320763E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.290235E-02 | lm-token-buffer-mib: 2.327942E+02 | global_load_balancing_loss: 9.999794E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11169.096 | params rms: 0.060477 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:08:54.392971 | finish at 2026-10-17 18:45:34 + [2026-10-15 11:40:26] iteration 1492/ 2384 | consumed samples: 47744 | elapsed time per iteration (ms): 226445.0 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 74089.5 | valid tokens/s: 74089.5 | physical tokens/s: 75508.1 | loss tokens/s: 53744.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.542030E-01 | lm-global-supervised-tokens: 1.217027E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.924137E-02 | lm-token-buffer-mib: 2.371265E+02 | global_load_balancing_loss: 1.000417E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11169.046 | params rms: 0.060477 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:06:28.977158 | finish at 2026-10-17 19:46:55 + [2026-10-15 11:44:13] iteration 1493/ 2384 | consumed samples: 47776 | elapsed time per iteration (ms): 226557.0 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 74052.9 | valid tokens/s: 74052.9 | physical tokens/s: 74567.1 | loss tokens/s: 58261.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.170213E-01 | lm-global-supervised-tokens: 1.319961E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.151098E-02 | lm-token-buffer-mib: 2.338696E+02 | global_load_balancing_loss: 1.000917E+00 | loss scale: 1.0 | grad norm: 0.227 | params norm: 11168.994 | params rms: 0.060476 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:04:22.300313 | finish at 2026-10-17 19:48:35 + [2026-10-15 11:47:54] iteration 1494/ 2384 | consumed samples: 47808 | elapsed time per iteration (ms): 221568.8 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 75719.8 | valid tokens/s: 75719.8 | physical tokens/s: 76628.3 | loss tokens/s: 55371.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.883456E-01 | lm-global-supervised-tokens: 1.226863E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.111292E-02 | lm-token-buffer-mib: 2.338828E+02 | global_load_balancing_loss: 1.001576E+00 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11168.944 | params rms: 0.060476 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:46:36.214364 | finish at 2026-10-17 18:34:30 + [2026-10-15 11:51:43] iteration 1495/ 2384 | consumed samples: 47840 | elapsed time per iteration (ms): 229145.0 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 73216.5 | valid tokens/s: 73216.5 | physical tokens/s: 74240.8 | loss tokens/s: 55795.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.012343E-01 | lm-global-supervised-tokens: 1.278522E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.629321E-02 | lm-token-buffer-mib: 2.366231E+02 | global_load_balancing_loss: 1.000503E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11168.898 | params rms: 0.060476 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:35:09.866619 | finish at 2026-10-17 20:26:53 + [2026-10-15 11:55:25] iteration 1496/ 2384 | consumed samples: 47872 | elapsed time per iteration (ms): 221846.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75625.3 | valid tokens/s: 75625.3 | physical tokens/s: 76027.1 | loss tokens/s: 57309.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.073849E-01 | lm-global-supervised-tokens: 1.271392E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.380838E-02 | lm-token-buffer-mib: 2.336673E+02 | global_load_balancing_loss: 1.000404E+00 | loss scale: 1.0 | grad norm: 0.239 | params norm: 11168.852 | params rms: 0.060476 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:43:19.848810 | finish at 2026-10-17 18:38:45 + [2026-10-15 11:59:12] iteration 1497/ 2384 | consumed samples: 47904 | elapsed time per iteration (ms): 227008.1 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 73905.8 | valid tokens/s: 73905.8 | physical tokens/s: 75080.9 | loss tokens/s: 52329.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.148738E-01 | lm-global-supervised-tokens: 1.187918E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.637309E-02 | lm-token-buffer-mib: 2.389852E+02 | global_load_balancing_loss: 9.999322E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11168.809 | params rms: 0.060475 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:55:56.143491 | finish at 2026-10-17 19:55:08 + [2026-10-15 12:02:56] iteration 1498/ 2384 | consumed samples: 47936 | elapsed time per iteration (ms): 223691.9 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.96% | FLOPs estimator: workload-aware | logical tokens/s: 75001.4 | valid tokens/s: 75001.4 | physical tokens/s: 76488.6 | loss tokens/s: 55893.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.452466E-01 | lm-global-supervised-tokens: 1.250293E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.026045E-02 | lm-token-buffer-mib: 2.389145E+02 | global_load_balancing_loss: 9.999400E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11168.766 | params rms: 0.060475 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:03:11.026582 | finish at 2026-10-17 19:06:07 + [2026-10-15 12:06:37] iteration 1499/ 2384 | consumed samples: 47968 | elapsed time per iteration (ms): 221169.6 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 75856.5 | valid tokens/s: 75856.5 | physical tokens/s: 76475.5 | loss tokens/s: 56151.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.730292E-01 | lm-global-supervised-tokens: 1.241901E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.539236E-02 | lm-token-buffer-mib: 2.349169E+02 | global_load_balancing_loss: 9.995831E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11168.721 | params rms: 0.060475 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:22:15.133356 | finish at 2026-10-17 18:28:52 + [2026-10-15 12:10:43] iteration 1500/ 2384 | consumed samples: 48000 | elapsed time per iteration (ms): 246301.6 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.67% | FLOPs estimator: workload-aware | logical tokens/s: 68116.5 | valid tokens/s: 68116.5 | physical tokens/s: 68670.5 | loss tokens/s: 52607.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.217322E-01 | lm-global-supervised-tokens: 1.295739E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.142940E-02 | lm-token-buffer-mib: 2.324032E+02 | global_load_balancing_loss: 9.996913E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11168.673 | params rms: 0.060475 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:28:50.596889 | finish at 2026-10-18 00:39:34 + [2026-10-15 12:14:25] iteration 1501/ 2384 | consumed samples: 48032 | elapsed time per iteration (ms): 221954.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75588.5 | valid tokens/s: 75588.5 | physical tokens/s: 76480.1 | loss tokens/s: 57010.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.513578E-01 | lm-global-supervised-tokens: 1.265364E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.216917E-02 | lm-token-buffer-mib: 2.345889E+02 | global_load_balancing_loss: 1.001475E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11168.623 | params rms: 0.060474 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:26:25.782413 | finish at 2026-10-17 18:40:51 + [2026-10-15 12:18:06] iteration 1502/ 2384 | consumed samples: 48064 | elapsed time per iteration (ms): 220374.3 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.16% | FLOPs estimator: workload-aware | logical tokens/s: 76130.5 | valid tokens/s: 76130.5 | physical tokens/s: 76988.1 | loss tokens/s: 56008.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.336543E-01 | lm-global-supervised-tokens: 1.234293E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.290879E-02 | lm-token-buffer-mib: 2.328305E+02 | global_load_balancing_loss: 1.000770E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11168.575 | params rms: 0.060474 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:59:30.132182 | finish at 2026-10-17 18:17:36 + [2026-10-15 12:21:50] iteration 1503/ 2384 | consumed samples: 48096 | elapsed time per iteration (ms): 224049.7 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.80% | FLOPs estimator: workload-aware | logical tokens/s: 74881.4 | valid tokens/s: 74881.4 | physical tokens/s: 75568.7 | loss tokens/s: 58711.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.665882E-01 | lm-global-supervised-tokens: 1.315422E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.745053E-02 | lm-token-buffer-mib: 2.332622E+02 | global_load_balancing_loss: 1.000441E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11168.530 | params rms: 0.060474 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:49:47.785089 | finish at 2026-10-17 19:11:38 + [2026-10-15 12:25:30] iteration 1504/ 2384 | consumed samples: 48128 | elapsed time per iteration (ms): 220556.4 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 76067.6 | valid tokens/s: 76067.6 | physical tokens/s: 76477.7 | loss tokens/s: 58097.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.667706E-01 | lm-global-supervised-tokens: 1.281375E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.624940E-02 | lm-token-buffer-mib: 2.328988E+02 | global_load_balancing_loss: 1.001419E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11168.483 | params rms: 0.060474 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:54:49.601212 | finish at 2026-10-17 18:20:20 + [2026-10-15 12:29:12] iteration 1505/ 2384 | consumed samples: 48160 | elapsed time per iteration (ms): 221824.6 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 75632.8 | valid tokens/s: 75632.8 | physical tokens/s: 76007.0 | loss tokens/s: 57522.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.784745E-01 | lm-global-supervised-tokens: 1.275984E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.519191E-02 | lm-token-buffer-mib: 2.330872E+02 | global_load_balancing_loss: 1.001779E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11168.441 | params rms: 0.060473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:09:43.866974 | finish at 2026-10-17 18:38:56 + [2026-10-15 12:32:56] iteration 1506/ 2384 | consumed samples: 48192 | elapsed time per iteration (ms): 223808.7 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.61% | FLOPs estimator: workload-aware | logical tokens/s: 74962.2 | valid tokens/s: 74962.2 | physical tokens/s: 75345.8 | loss tokens/s: 54149.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.176627E-01 | lm-global-supervised-tokens: 1.211922E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.489012E-02 | lm-token-buffer-mib: 2.334510E+02 | global_load_balancing_loss: 9.997964E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11168.400 | params rms: 0.060473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:35:04.025905 | finish at 2026-10-17 19:08:00 + [2026-10-15 12:36:40] iteration 1507/ 2384 | consumed samples: 48224 | elapsed time per iteration (ms): 223582.5 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.72% | FLOPs estimator: workload-aware | logical tokens/s: 75038.0 | valid tokens/s: 75038.0 | physical tokens/s: 75468.1 | loss tokens/s: 57772.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.880169E-01 | lm-global-supervised-tokens: 1.291695E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.501165E-02 | lm-token-buffer-mib: 2.330342E+02 | global_load_balancing_loss: 9.986320E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11168.359 | params rms: 0.060473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:28:01.831365 | finish at 2026-10-17 19:04:41 + [2026-10-15 12:40:23] iteration 1508/ 2384 | consumed samples: 48256 | elapsed time per iteration (ms): 223065.8 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 75211.9 | valid tokens/s: 75211.9 | physical tokens/s: 75652.6 | loss tokens/s: 55564.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.936148E-01 | lm-global-supervised-tokens: 1.239462E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.263038E-02 | lm-token-buffer-mib: 2.339139E+02 | global_load_balancing_loss: 9.978765E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11168.316 | params rms: 0.060473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:16:45.654542 | finish at 2026-10-17 18:57:08 + [2026-10-15 12:44:06] iteration 1509/ 2384 | consumed samples: 48288 | elapsed time per iteration (ms): 223761.8 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 74977.9 | valid tokens/s: 74977.9 | physical tokens/s: 75699.9 | loss tokens/s: 57724.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.172156E-01 | lm-global-supervised-tokens: 1.291662E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.559815E-02 | lm-token-buffer-mib: 2.351353E+02 | global_load_balancing_loss: 9.996821E-01 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11168.273 | params rms: 0.060473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:23:11.576922 | finish at 2026-10-17 19:07:18 + [2026-10-15 12:48:14] iteration 1510/ 2384 | consumed samples: 48320 | elapsed time per iteration (ms): 247544.4 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 67774.5 | valid tokens/s: 67774.5 | physical tokens/s: 68765.8 | loss tokens/s: 50431.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.994563E-01 | lm-global-supervised-tokens: 1.248393E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.429899E-02 | lm-token-buffer-mib: 2.351345E+02 | global_load_balancing_loss: 1.001064E+00 | loss scale: 1.0 | grad norm: 0.257 | params norm: 11168.234 | params rms: 0.060472 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:05:53.787868 | finish at 2026-10-18 00:54:08 + [2026-10-15 12:51:56] iteration 1511/ 2384 | consumed samples: 48352 | elapsed time per iteration (ms): 222583.7 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.21% | FLOPs estimator: workload-aware | logical tokens/s: 75374.7 | valid tokens/s: 75374.7 | physical tokens/s: 75907.6 | loss tokens/s: 60177.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.915797E-01 | lm-global-supervised-tokens: 1.339463E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.299297E-02 | lm-token-buffer-mib: 2.334943E+02 | global_load_balancing_loss: 1.001622E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11168.196 | params rms: 0.060472 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:58:35.603143 | finish at 2026-10-17 18:50:32 + [2026-10-15 12:55:35] iteration 1512/ 2384 | consumed samples: 48384 | elapsed time per iteration (ms): 218608.6 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 76745.4 | valid tokens/s: 76745.4 | physical tokens/s: 77845.4 | loss tokens/s: 61221.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.909652E-01 | lm-global-supervised-tokens: 1.338350E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.449134E-02 | lm-token-buffer-mib: 2.335104E+02 | global_load_balancing_loss: 1.001246E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11168.159 | params rms: 0.060472 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:57:06.661900 | finish at 2026-10-17 17:52:42 + [2026-10-15 12:59:16] iteration 1513/ 2384 | consumed samples: 48416 | elapsed time per iteration (ms): 220976.8 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 75922.9 | valid tokens/s: 75922.9 | physical tokens/s: 76844.5 | loss tokens/s: 58234.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.587219E-01 | lm-global-supervised-tokens: 1.286856E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.423687E-02 | lm-token-buffer-mib: 2.330499E+02 | global_load_balancing_loss: 1.001608E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11168.122 | params rms: 0.060472 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:27:50.819558 | finish at 2026-10-17 18:27:07 + [2026-10-15 13:02:57] iteration 1514/ 2384 | consumed samples: 48448 | elapsed time per iteration (ms): 220891.5 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 75952.1 | valid tokens/s: 75952.1 | physical tokens/s: 76407.2 | loss tokens/s: 53610.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.205671E-01 | lm-global-supervised-tokens: 1.184217E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.439133E-02 | lm-token-buffer-mib: 2.333573E+02 | global_load_balancing_loss: 1.001346E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11168.083 | params rms: 0.060472 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:22:55.632584 | finish at 2026-10-17 18:25:53 + [2026-10-15 13:06:40] iteration 1515/ 2384 | consumed samples: 48480 | elapsed time per iteration (ms): 222660.1 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.93% | FLOPs estimator: workload-aware | logical tokens/s: 75348.9 | valid tokens/s: 75348.9 | physical tokens/s: 76363.1 | loss tokens/s: 53875.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.474756E-01 | lm-global-supervised-tokens: 1.199584E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.057731E-02 | lm-token-buffer-mib: 2.334580E+02 | global_load_balancing_loss: 1.002188E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11168.042 | params rms: 0.060471 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:44:51.633723 | finish at 2026-10-17 18:51:31 + [2026-10-15 13:10:24] iteration 1516/ 2384 | consumed samples: 48512 | elapsed time per iteration (ms): 223931.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 74921.1 | valid tokens/s: 74921.1 | physical tokens/s: 75851.0 | loss tokens/s: 58767.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.677670E-01 | lm-global-supervised-tokens: 1.315994E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.470214E-02 | lm-token-buffer-mib: 2.359654E+02 | global_load_balancing_loss: 9.981815E-01 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11168.001 | params rms: 0.060471 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:59:32.184359 | finish at 2026-10-17 19:09:56 + [2026-10-15 13:14:05] iteration 1517/ 2384 | consumed samples: 48544 | elapsed time per iteration (ms): 221011.5 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 75910.9 | valid tokens/s: 75910.9 | physical tokens/s: 76432.6 | loss tokens/s: 57011.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.969502E-01 | lm-global-supervised-tokens: 1.260020E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.656759E-02 | lm-token-buffer-mib: 2.344779E+02 | global_load_balancing_loss: 9.981499E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11167.963 | params rms: 0.060471 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:13:36.975358 | finish at 2026-10-17 18:27:42 + [2026-10-15 13:17:50] iteration 1518/ 2384 | consumed samples: 48576 | elapsed time per iteration (ms): 225063.9 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 74544.2 | valid tokens/s: 74544.2 | physical tokens/s: 75882.2 | loss tokens/s: 57522.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.274447E-01 | lm-global-supervised-tokens: 1.294612E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.087097E-02 | lm-token-buffer-mib: 2.355746E+02 | global_load_balancing_loss: 9.999256E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11167.924 | params rms: 0.060471 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:08:25.336982 | finish at 2026-10-17 19:26:15 + [2026-10-15 13:21:34] iteration 1519/ 2384 | consumed samples: 48608 | elapsed time per iteration (ms): 224026.1 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 74889.5 | valid tokens/s: 74889.5 | physical tokens/s: 75625.2 | loss tokens/s: 52444.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.499135E-01 | lm-global-supervised-tokens: 1.174890E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.286729E-02 | lm-token-buffer-mib: 2.357897E+02 | global_load_balancing_loss: 1.000514E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11167.886 | params rms: 0.060470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:49:42.575194 | finish at 2026-10-17 19:11:16 + [2026-10-15 13:25:37] iteration 1520/ 2384 | consumed samples: 48640 | elapsed time per iteration (ms): 243818.9 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 68810.0 | valid tokens/s: 68810.0 | physical tokens/s: 69222.2 | loss tokens/s: 50036.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.857533E-01 | lm-global-supervised-tokens: 1.219975E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.597598E-02 | lm-token-buffer-mib: 2.330747E+02 | global_load_balancing_loss: 1.000217E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11167.845 | params rms: 0.060470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:30:59.569450 | finish at 2026-10-17 23:56:37 +times across ranks (ms): + save-checkpoint: + rank 0: 24981.93 + rank 1: 24981.99 + rank 2: 24981.98 + rank 3: 24981.97 + rank 4: 24981.91 + rank 5: 24981.95 + rank 6: 24981.92 + rank 7: 24981.91 + rank 8: 24981.91 + rank 9: 24981.92 + rank 10: 24981.92 + rank 11: 24981.91 + rank 12: 24981.88 + rank 13: 24981.92 + rank 14: 24981.89 + rank 15: 24981.89 + rank 16: 24981.91 + rank 17: 24981.88 + rank 18: 24981.89 + rank 19: 24981.92 + rank 20: 24981.85 + rank 21: 24981.86 + rank 22: 24981.88 + rank 23: 24981.89 + rank 24: 24981.96 + rank 25: 24981.95 + rank 26: 24981.98 + rank 27: 24981.97 + rank 28: 24981.83 + rank 29: 24981.84 + rank 30: 24981.98 + rank 31: 24981.93 + rank 32: 24980.27 + rank 33: 24980.28 + rank 34: 24980.25 + rank 35: 24980.27 + rank 36: 24980.23 + rank 37: 24980.26 + rank 38: 24980.25 + rank 39: 24980.27 + rank 40: 24983.52 + rank 41: 24983.49 + rank 42: 24983.49 + rank 43: 24983.51 + rank 44: 24983.62 + rank 45: 24983.48 + rank 46: 24983.50 + rank 47: 24983.50 + rank 48: 24982.54 + rank 49: 24982.53 + rank 50: 24982.53 + rank 51: 24982.59 + rank 52: 24982.47 + rank 53: 24982.50 + rank 54: 24982.47 + rank 55: 24982.52 + rank 56: 24981.93 + rank 57: 24981.93 + rank 58: 24981.95 + rank 59: 24981.92 + rank 60: 24981.96 + rank 61: 24981.89 + rank 62: 24981.90 + rank 63: 24981.90 + [2026-10-15 13:29:44] iteration 1521/ 2384 | consumed samples: 48672 | elapsed time per iteration (ms): 221940.0 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 75593.4 | valid tokens/s: 75593.4 | physical tokens/s: 76332.3 | loss tokens/s: 57645.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.485318E-01 | lm-global-supervised-tokens: 1.279379E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.394714E-02 | lm-token-buffer-mib: 2.339561E+02 | global_load_balancing_loss: 9.994969E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11167.801 | params rms: 0.060470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:12:14.211613 | finish at 2026-10-17 18:41:59 + [2026-10-15 13:33:32] iteration 1522/ 2384 | consumed samples: 48704 | elapsed time per iteration (ms): 227936.3 | throughput per GPU (TFLOP/s/GPU): 18.4 | MFU 5.90% | FLOPs estimator: workload-aware | logical tokens/s: 73604.8 | valid tokens/s: 73604.8 | physical tokens/s: 74400.9 | loss tokens/s: 57271.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.536387E-01 | lm-global-supervised-tokens: 1.305417E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.316000E-02 | lm-token-buffer-mib: 2.354730E+02 | global_load_balancing_loss: 1.000396E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11167.761 | params rms: 0.060470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:34:41.097416 | finish at 2026-10-17 20:08:13 + [2026-10-15 13:37:14] iteration 1523/ 2384 | consumed samples: 48736 | elapsed time per iteration (ms): 221128.0 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 75870.8 | valid tokens/s: 75870.8 | physical tokens/s: 76288.4 | loss tokens/s: 53951.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.928665E-01 | lm-global-supervised-tokens: 1.193011E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.284360E-02 | lm-token-buffer-mib: 2.326584E+02 | global_load_balancing_loss: 1.001567E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11167.722 | params rms: 0.060470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:53:11.173942 | finish at 2026-10-17 18:30:25 + [2026-10-15 13:40:56] iteration 1524/ 2384 | consumed samples: 48768 | elapsed time per iteration (ms): 222754.2 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75317.1 | valid tokens/s: 75317.1 | physical tokens/s: 76373.1 | loss tokens/s: 56360.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.462935E-01 | lm-global-supervised-tokens: 1.255448E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.334862E-02 | lm-token-buffer-mib: 2.369706E+02 | global_load_balancing_loss: 1.001440E+00 | loss scale: 1.0 | grad norm: 0.349 | params norm: 11167.680 | params rms: 0.060469 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:12:48.586969 | finish at 2026-10-17 18:53:45 + [2026-10-15 13:44:35] iteration 1525/ 2384 | consumed samples: 48800 | elapsed time per iteration (ms): 218408.9 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 76815.6 | valid tokens/s: 76815.6 | physical tokens/s: 77176.0 | loss tokens/s: 61349.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.122403E-01 | lm-global-supervised-tokens: 1.339921E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.969311E-02 | lm-token-buffer-mib: 2.329475E+02 | global_load_balancing_loss: 1.001016E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11167.638 | params rms: 0.060469 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:06:53.267852 | finish at 2026-10-17 17:51:28 + [2026-10-15 13:48:17] iteration 1526/ 2384 | consumed samples: 48832 | elapsed time per iteration (ms): 222228.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75495.2 | valid tokens/s: 75495.2 | physical tokens/s: 75999.6 | loss tokens/s: 57820.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.018999E-01 | lm-global-supervised-tokens: 1.284932E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.065392E-02 | lm-token-buffer-mib: 2.340852E+02 | global_load_balancing_loss: 9.993243E-01 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11167.599 | params rms: 0.060469 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:57:52.023653 | finish at 2026-10-17 18:46:09 + [2026-10-15 13:51:59] iteration 1527/ 2384 | consumed samples: 48864 | elapsed time per iteration (ms): 221836.5 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 75628.6 | valid tokens/s: 75628.6 | physical tokens/s: 76719.0 | loss tokens/s: 59595.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.013068E-01 | lm-global-supervised-tokens: 1.322052E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.424866E-02 | lm-token-buffer-mib: 2.350081E+02 | global_load_balancing_loss: 9.990787E-01 | loss scale: 1.0 | grad norm: 0.483 | params norm: 11167.557 | params rms: 0.060469 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:48:33.856125 | finish at 2026-10-17 18:40:33 + [2026-10-15 13:55:41] iteration 1528/ 2384 | consumed samples: 48896 | elapsed time per iteration (ms): 222665.2 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75347.1 | valid tokens/s: 75347.1 | physical tokens/s: 75749.4 | loss tokens/s: 55258.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.178122E-01 | lm-global-supervised-tokens: 1.230417E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.592743E-02 | lm-token-buffer-mib: 2.338348E+02 | global_load_balancing_loss: 9.994106E-01 | loss scale: 1.0 | grad norm: 0.217 | params norm: 11167.515 | params rms: 0.060468 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:56:41.387930 | finish at 2026-10-17 18:52:23 + [2026-10-15 13:59:26] iteration 1529/ 2384 | consumed samples: 48928 | elapsed time per iteration (ms): 224168.6 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 74841.8 | valid tokens/s: 74841.8 | physical tokens/s: 75541.2 | loss tokens/s: 56460.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.969153E-01 | lm-global-supervised-tokens: 1.265660E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.174188E-02 | lm-token-buffer-mib: 2.335046E+02 | global_load_balancing_loss: 1.001307E+00 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11167.478 | params rms: 0.060468 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:14:24.130647 | finish at 2026-10-17 19:13:50 + [2026-10-15 14:03:27] iteration 1530/ 2384 | consumed samples: 48960 | elapsed time per iteration (ms): 241430.7 | throughput per GPU (TFLOP/s/GPU): 16.8 | MFU 5.37% | FLOPs estimator: workload-aware | logical tokens/s: 69490.8 | valid tokens/s: 69490.8 | physical tokens/s: 70488.1 | loss tokens/s: 53731.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.882491E-01 | lm-global-supervised-tokens: 1.297244E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.344407E-02 | lm-token-buffer-mib: 2.334303E+02 | global_load_balancing_loss: 1.000739E+00 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11167.438 | params rms: 0.060468 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:16:21.797493 | finish at 2026-10-17 23:19:49 + [2026-10-15 14:07:17] iteration 1531/ 2384 | consumed samples: 48992 | elapsed time per iteration (ms): 229654.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 73054.2 | valid tokens/s: 73054.2 | physical tokens/s: 73611.4 | loss tokens/s: 57744.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.043787E-01 | lm-global-supervised-tokens: 1.326114E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.777834E-02 | lm-token-buffer-mib: 2.340415E+02 | global_load_balancing_loss: 1.000651E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11167.401 | params rms: 0.060468 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:24:54.987111 | finish at 2026-10-17 20:32:12 + [2026-10-15 14:10:59] iteration 1532/ 2384 | consumed samples: 49024 | elapsed time per iteration (ms): 221898.2 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.02% | FLOPs estimator: workload-aware | logical tokens/s: 75607.6 | valid tokens/s: 75607.6 | physical tokens/s: 76302.8 | loss tokens/s: 58055.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.465775E-01 | lm-global-supervised-tokens: 1.288232E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.280433E-02 | lm-token-buffer-mib: 2.333306E+02 | global_load_balancing_loss: 1.002579E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11167.363 | params rms: 0.060468 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:30:57.277805 | finish at 2026-10-17 18:41:56 + [2026-10-15 14:14:42] iteration 1533/ 2384 | consumed samples: 49056 | elapsed time per iteration (ms): 223920.8 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 74924.6 | valid tokens/s: 74924.6 | physical tokens/s: 75460.7 | loss tokens/s: 57009.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.197387E-01 | lm-global-supervised-tokens: 1.276555E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.265209E-02 | lm-token-buffer-mib: 2.345513E+02 | global_load_balancing_loss: 1.000259E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11167.324 | params rms: 0.060467 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:55:56.621673 | finish at 2026-10-17 19:10:39 + [2026-10-15 14:18:29] iteration 1534/ 2384 | consumed samples: 49088 | elapsed time per iteration (ms): 226180.2 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 74176.2 | valid tokens/s: 74176.1 | physical tokens/s: 74701.2 | loss tokens/s: 58648.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.524489E-01 | lm-global-supervised-tokens: 1.326512E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.376973E-02 | lm-token-buffer-mib: 2.329251E+02 | global_load_balancing_loss: 9.998032E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11167.284 | params rms: 0.060467 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:24:13.173935 | finish at 2026-10-17 19:42:42 + [2026-10-15 14:22:15] iteration 1535/ 2384 | consumed samples: 49120 | elapsed time per iteration (ms): 226305.1 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 74135.2 | valid tokens/s: 74135.2 | physical tokens/s: 75508.1 | loss tokens/s: 53669.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.187883E-01 | lm-global-supervised-tokens: 1.214573E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.290700E-02 | lm-token-buffer-mib: 2.369039E+02 | global_load_balancing_loss: 1.000015E+00 | loss scale: 1.0 | grad norm: 0.237 | params norm: 11167.247 | params rms: 0.060467 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:22:13.004972 | finish at 2026-10-17 19:44:28 + [2026-10-15 14:25:57] iteration 1536/ 2384 | consumed samples: 49152 | elapsed time per iteration (ms): 222347.7 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 75454.7 | valid tokens/s: 75454.7 | physical tokens/s: 76278.1 | loss tokens/s: 57573.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.856784E-01 | lm-global-supervised-tokens: 1.280126E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.323561E-02 | lm-token-buffer-mib: 2.339804E+02 | global_load_balancing_loss: 9.996239E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11167.209 | params rms: 0.060467 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:22:30.890541 | finish at 2026-10-17 18:48:28 + [2026-10-15 14:29:40] iteration 1537/ 2384 | consumed samples: 49184 | elapsed time per iteration (ms): 222532.4 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75392.2 | valid tokens/s: 75392.2 | physical tokens/s: 76091.7 | loss tokens/s: 59596.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.842818E-01 | lm-global-supervised-tokens: 1.326222E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.170606E-02 | lm-token-buffer-mib: 2.333602E+02 | global_load_balancing_loss: 9.997900E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11167.170 | params rms: 0.060467 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:21:24.980068 | finish at 2026-10-17 18:51:05 + [2026-10-15 14:33:19] iteration 1538/ 2384 | consumed samples: 49216 | elapsed time per iteration (ms): 219617.6 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.01% | FLOPs estimator: workload-aware | logical tokens/s: 76392.7 | valid tokens/s: 76392.7 | physical tokens/s: 77349.9 | loss tokens/s: 58721.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.606739E-01 | lm-global-supervised-tokens: 1.289625E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.176157E-02 | lm-token-buffer-mib: 2.356595E+02 | global_load_balancing_loss: 1.000156E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11167.128 | params rms: 0.060466 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:36:36.497033 | finish at 2026-10-17 18:09:56 + [2026-10-15 14:37:00] iteration 1539/ 2384 | consumed samples: 49248 | elapsed time per iteration (ms): 220585.7 | throughput per GPU (TFLOP/s/GPU): 27.0 | MFU 8.65% | FLOPs estimator: workload-aware | logical tokens/s: 76057.4 | valid tokens/s: 76057.4 | physical tokens/s: 76469.9 | loss tokens/s: 56833.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.237867E-01 | lm-global-supervised-tokens: 1.253654E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.251626E-02 | lm-token-buffer-mib: 2.339734E+02 | global_load_balancing_loss: 9.989996E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11167.088 | params rms: 0.060466 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:46:34.941108 | finish at 2026-10-17 18:23:35 + [2026-10-15 14:41:03] iteration 1540/ 2384 | consumed samples: 49280 | elapsed time per iteration (ms): 243012.7 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 69038.3 | valid tokens/s: 69038.3 | physical tokens/s: 69478.0 | loss tokens/s: 50238.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.909902E-01 | lm-global-supervised-tokens: 1.220856E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.632394E-02 | lm-token-buffer-mib: 2.327770E+02 | global_load_balancing_loss: 1.001908E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11167.047 | params rms: 0.060466 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:58:22.707197 | finish at 2026-10-17 23:39:26 + [2026-10-15 14:44:44] iteration 1541/ 2384 | consumed samples: 49312 | elapsed time per iteration (ms): 221251.1 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 75828.7 | valid tokens/s: 75828.7 | physical tokens/s: 76774.5 | loss tokens/s: 58456.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.855507E-01 | lm-global-supervised-tokens: 1.293362E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.361589E-02 | lm-token-buffer-mib: 2.343214E+02 | global_load_balancing_loss: 1.001127E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11167.005 | params rms: 0.060466 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:48:34.656250 | finish at 2026-10-17 18:33:19 + [2026-10-15 14:48:25] iteration 1542/ 2384 | consumed samples: 49344 | elapsed time per iteration (ms): 220714.0 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 76013.3 | valid tokens/s: 76013.3 | physical tokens/s: 76830.5 | loss tokens/s: 58364.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.218228E-01 | lm-global-supervised-tokens: 1.288191E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.777531E-02 | lm-token-buffer-mib: 2.334861E+02 | global_load_balancing_loss: 1.000384E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11166.962 | params rms: 0.060465 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:37:21.179557 | finish at 2026-10-17 18:25:46 + [2026-10-15 14:52:08] iteration 1543/ 2384 | consumed samples: 49376 | elapsed time per iteration (ms): 222707.6 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.14% | FLOPs estimator: workload-aware | logical tokens/s: 75332.9 | valid tokens/s: 75332.9 | physical tokens/s: 76021.2 | loss tokens/s: 60061.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.865418E-01 | lm-global-supervised-tokens: 1.337615E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.521299E-02 | lm-token-buffer-mib: 2.341692E+02 | global_load_balancing_loss: 1.001560E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11166.919 | params rms: 0.060465 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:01:37.073853 | finish at 2026-10-17 18:53:45 + [2026-10-15 14:55:50] iteration 1544/ 2384 | consumed samples: 49408 | elapsed time per iteration (ms): 222089.5 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.19% | FLOPs estimator: workload-aware | logical tokens/s: 75542.6 | valid tokens/s: 75542.6 | physical tokens/s: 76548.0 | loss tokens/s: 57993.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.214529E-01 | lm-global-supervised-tokens: 1.287973E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.300003E-02 | lm-token-buffer-mib: 2.365747E+02 | global_load_balancing_loss: 9.999098E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11166.876 | params rms: 0.060465 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:49:15.148916 | finish at 2026-10-17 18:45:05 + [2026-10-15 14:59:34] iteration 1545/ 2384 | consumed samples: 49440 | elapsed time per iteration (ms): 224308.0 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.09% | FLOPs estimator: workload-aware | logical tokens/s: 74795.3 | valid tokens/s: 74795.3 | physical tokens/s: 75037.1 | loss tokens/s: 58410.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.909991E-01 | lm-global-supervised-tokens: 1.310185E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.563028E-02 | lm-token-buffer-mib: 2.328483E+02 | global_load_balancing_loss: 9.989529E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11166.835 | params rms: 0.060465 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:16:34.439278 | finish at 2026-10-17 19:16:09 + [2026-10-15 15:03:19] iteration 1546/ 2384 | consumed samples: 49472 | elapsed time per iteration (ms): 224515.0 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 74726.4 | valid tokens/s: 74726.4 | physical tokens/s: 75483.3 | loss tokens/s: 55288.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.786119E-01 | lm-global-supervised-tokens: 1.241306E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.104067E-02 | lm-token-buffer-mib: 2.350689E+02 | global_load_balancing_loss: 1.001202E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11166.798 | params rms: 0.060465 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:15:43.530529 | finish at 2026-10-17 19:19:02 + [2026-10-15 15:07:08] iteration 1547/ 2384 | consumed samples: 49504 | elapsed time per iteration (ms): 229357.1 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 73148.8 | valid tokens/s: 73148.8 | physical tokens/s: 74411.5 | loss tokens/s: 55016.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.308621E-01 | lm-global-supervised-tokens: 1.261844E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.098169E-02 | lm-token-buffer-mib: 2.385493E+02 | global_load_balancing_loss: 1.002582E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11166.763 | params rms: 0.060464 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:19:31.882331 | finish at 2026-10-17 20:26:40 + [2026-10-15 15:10:52] iteration 1548/ 2384 | consumed samples: 49536 | elapsed time per iteration (ms): 223907.3 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 74929.3 | valid tokens/s: 74929.2 | physical tokens/s: 75248.6 | loss tokens/s: 55099.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.988708E-01 | lm-global-supervised-tokens: 1.233710E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.260407E-02 | lm-token-buffer-mib: 2.324931E+02 | global_load_balancing_loss: 1.001196E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11166.723 | params rms: 0.060464 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:59:46.502398 | finish at 2026-10-17 19:10:38 + [2026-10-15 15:14:35] iteration 1549/ 2384 | consumed samples: 49568 | elapsed time per iteration (ms): 222579.3 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.19% | FLOPs estimator: workload-aware | logical tokens/s: 75376.2 | valid tokens/s: 75376.2 | physical tokens/s: 76152.4 | loss tokens/s: 57166.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.936522E-01 | lm-global-supervised-tokens: 1.272401E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.292804E-02 | lm-token-buffer-mib: 2.348355E+02 | global_load_balancing_loss: 1.000539E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11166.684 | params rms: 0.060464 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:37:33.694137 | finish at 2026-10-17 18:52:08 + [2026-10-15 15:18:37] iteration 1550/ 2384 | consumed samples: 49600 | elapsed time per iteration (ms): 242445.0 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 69199.9 | valid tokens/s: 69199.9 | physical tokens/s: 69622.5 | loss tokens/s: 47629.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.967690E-01 | lm-global-supervised-tokens: 1.154744E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.350502E-02 | lm-token-buffer-mib: 2.331215E+02 | global_load_balancing_loss: 9.996024E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11166.651 | params rms: 0.060464 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:09:59.104294 | finish at 2026-10-17 23:28:36 + [2026-10-15 15:22:16] iteration 1551/ 2384 | consumed samples: 49632 | elapsed time per iteration (ms): 219417.9 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 76462.3 | valid tokens/s: 76462.3 | physical tokens/s: 77301.3 | loss tokens/s: 56220.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.330567E-01 | lm-global-supervised-tokens: 1.233567E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.337673E-02 | lm-token-buffer-mib: 2.337776E+02 | global_load_balancing_loss: 9.988915E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11166.618 | params rms: 0.060464 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:46:15.098259 | finish at 2026-10-17 18:08:31 + [2026-10-15 15:25:57] iteration 1552/ 2384 | consumed samples: 49664 | elapsed time per iteration (ms): 221000.7 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 75914.7 | valid tokens/s: 75914.7 | physical tokens/s: 76396.4 | loss tokens/s: 55579.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.792568E-01 | lm-global-supervised-tokens: 1.228304E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.091434E-02 | lm-token-buffer-mib: 2.334605E+02 | global_load_balancing_loss: 9.984958E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11166.587 | params rms: 0.060463 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:04:32.577835 | finish at 2026-10-17 18:30:30 + [2026-10-15 15:29:38] iteration 1553/ 2384 | consumed samples: 49696 | elapsed time per iteration (ms): 220917.9 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.26% | FLOPs estimator: workload-aware | logical tokens/s: 75943.2 | valid tokens/s: 75943.2 | physical tokens/s: 76941.2 | loss tokens/s: 57965.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.490361E-01 | lm-global-supervised-tokens: 1.280558E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.242990E-02 | lm-token-buffer-mib: 2.356334E+02 | global_load_balancing_loss: 9.994065E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11166.554 | params rms: 0.060463 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:59:42.804690 | finish at 2026-10-17 18:29:21 + [2026-10-15 15:33:22] iteration 1554/ 2384 | consumed samples: 49728 | elapsed time per iteration (ms): 223632.2 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 75021.2 | valid tokens/s: 75021.2 | physical tokens/s: 75797.9 | loss tokens/s: 57804.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.673818E-01 | lm-global-supervised-tokens: 1.292684E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.486348E-02 | lm-token-buffer-mib: 2.347209E+02 | global_load_balancing_loss: 1.000253E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11166.518 | params rms: 0.060463 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:33:34.749749 | finish at 2026-10-17 19:06:57 + [2026-10-15 15:37:03] iteration 1555/ 2384 | consumed samples: 49760 | elapsed time per iteration (ms): 220647.0 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 76036.3 | valid tokens/s: 76036.3 | physical tokens/s: 76709.7 | loss tokens/s: 58432.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.847624E-01 | lm-global-supervised-tokens: 1.289284E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.493057E-02 | lm-token-buffer-mib: 2.325037E+02 | global_load_balancing_loss: 1.002352E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11166.482 | params rms: 0.060463 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:48:36.390337 | finish at 2026-10-17 18:25:39 + [2026-10-15 15:40:48] iteration 1556/ 2384 | consumed samples: 49792 | elapsed time per iteration (ms): 225536.1 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 74388.0 | valid tokens/s: 74388.0 | physical tokens/s: 74950.9 | loss tokens/s: 57102.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.645027E-01 | lm-global-supervised-tokens: 1.287868E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.200202E-02 | lm-token-buffer-mib: 2.349961E+02 | global_load_balancing_loss: 1.000127E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11166.446 | params rms: 0.060463 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:52:23.887568 | finish at 2026-10-17 19:33:12 + [2026-10-15 15:44:41] iteration 1557/ 2384 | consumed samples: 49824 | elapsed time per iteration (ms): 232577.7 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.86% | FLOPs estimator: workload-aware | logical tokens/s: 72135.7 | valid tokens/s: 72135.7 | physical tokens/s: 73984.6 | loss tokens/s: 54843.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.584392E-01 | lm-global-supervised-tokens: 1.275545E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.073390E-02 | lm-token-buffer-mib: 2.412458E+02 | global_load_balancing_loss: 1.000968E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11166.412 | params rms: 0.060462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:25:41.788182 | finish at 2026-10-17 21:10:22 + [2026-10-15 15:48:21] iteration 1558/ 2384 | consumed samples: 49856 | elapsed time per iteration (ms): 219968.3 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 76271.0 | valid tokens/s: 76271.0 | physical tokens/s: 76995.8 | loss tokens/s: 54567.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.531222E-01 | lm-global-supervised-tokens: 1.200304E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.629366E-02 | lm-token-buffer-mib: 2.342145E+02 | global_load_balancing_loss: 1.001491E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11166.377 | params rms: 0.060462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:28:13.799541 | finish at 2026-10-17 18:16:34 + [2026-10-15 15:52:04] iteration 1559/ 2384 | consumed samples: 49888 | elapsed time per iteration (ms): 222991.7 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 75236.8 | valid tokens/s: 75236.8 | physical tokens/s: 75999.3 | loss tokens/s: 60621.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.398402E-01 | lm-global-supervised-tokens: 1.351797E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.087792E-02 | lm-token-buffer-mib: 2.332265E+02 | global_load_balancing_loss: 1.001302E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11166.343 | params rms: 0.060462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:06:08.114090 | finish at 2026-10-17 18:58:12 + [2026-10-15 15:56:10] iteration 1560/ 2384 | consumed samples: 49920 | elapsed time per iteration (ms): 246242.4 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.06% | FLOPs estimator: workload-aware | logical tokens/s: 68132.8 | valid tokens/s: 68132.8 | physical tokens/s: 68896.8 | loss tokens/s: 54065.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.197434E-01 | lm-global-supervised-tokens: 1.331318E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.561857E-02 | lm-token-buffer-mib: 2.337125E+02 | global_load_balancing_loss: 1.000135E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11166.312 | params rms: 0.060462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:21:43.703360 | finish at 2026-10-18 00:17:54 +times across ranks (ms): + save-checkpoint: + rank 0: 25025.90 + rank 1: 25025.92 + rank 2: 25025.93 + rank 3: 25025.93 + rank 4: 25025.90 + rank 5: 25025.90 + rank 6: 25025.89 + rank 7: 25025.88 + rank 8: 25025.77 + rank 9: 25025.81 + rank 10: 25025.82 + rank 11: 25025.77 + rank 12: 25025.80 + rank 13: 25025.77 + rank 14: 25025.79 + rank 15: 25025.79 + rank 16: 25026.29 + rank 17: 25026.30 + rank 18: 25026.31 + rank 19: 25026.27 + rank 20: 25026.32 + rank 21: 25026.26 + rank 22: 25026.32 + rank 23: 25026.28 + rank 24: 25026.17 + rank 25: 25026.18 + rank 26: 25026.04 + rank 27: 25026.37 + rank 28: 25026.11 + rank 29: 25026.81 + rank 30: 25026.20 + rank 31: 25025.90 + rank 32: 25025.89 + rank 33: 25026.08 + rank 34: 25025.92 + rank 35: 25025.92 + rank 36: 25025.86 + rank 37: 25025.94 + rank 38: 25025.86 + rank 39: 25025.95 + rank 40: 25025.94 + rank 41: 25025.90 + rank 42: 25025.91 + rank 43: 25025.95 + rank 44: 25025.88 + rank 45: 25025.88 + rank 46: 25025.88 + rank 47: 25025.89 + rank 48: 25026.12 + rank 49: 25026.17 + rank 50: 25026.14 + rank 51: 25026.21 + rank 52: 25026.17 + rank 53: 25026.28 + rank 54: 25026.23 + rank 55: 25026.24 + rank 56: 25026.26 + rank 57: 25026.29 + rank 58: 25026.29 + rank 59: 25026.28 + rank 60: 25026.32 + rank 61: 25026.26 + rank 62: 25026.31 + rank 63: 25026.26 + [2026-10-15 16:00:14] iteration 1561/ 2384 | consumed samples: 49952 | elapsed time per iteration (ms): 219235.6 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 76525.7 | valid tokens/s: 76525.7 | physical tokens/s: 77617.7 | loss tokens/s: 57362.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.863064E-01 | lm-global-supervised-tokens: 1.257593E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.295646E-01 | lm-token-buffer-mib: 2.365790E+02 | global_load_balancing_loss: 1.001211E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11166.283 | params rms: 0.060462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:07:10.924893 | finish at 2026-10-17 18:07:25 + [2026-10-15 16:04:02] iteration 1562/ 2384 | consumed samples: 49984 | elapsed time per iteration (ms): 227876.2 | throughput per GPU (TFLOP/s/GPU): 18.5 | MFU 5.93% | FLOPs estimator: workload-aware | logical tokens/s: 73624.2 | valid tokens/s: 73624.2 | physical tokens/s: 74480.6 | loss tokens/s: 59122.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.307088E-01 | lm-global-supervised-tokens: 1.347253E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.747115E-02 | lm-token-buffer-mib: 2.359142E+02 | global_load_balancing_loss: 1.000261E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11166.251 | params rms: 0.060462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:01:54.205991 | finish at 2026-10-17 20:05:56 + [2026-10-15 16:07:43] iteration 1563/ 2384 | consumed samples: 50016 | elapsed time per iteration (ms): 220885.8 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 75954.1 | valid tokens/s: 75954.1 | physical tokens/s: 76144.0 | loss tokens/s: 55256.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.463203E-01 | lm-global-supervised-tokens: 1.220530E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.181765E-02 | lm-token-buffer-mib: 2.327719E+02 | global_load_balancing_loss: 9.995975E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11166.219 | params rms: 0.060461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:22:27.246207 | finish at 2026-10-17 18:30:10 + [2026-10-15 16:11:27] iteration 1564/ 2384 | consumed samples: 50048 | elapsed time per iteration (ms): 223865.8 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 74943.0 | valid tokens/s: 74943.0 | physical tokens/s: 75854.9 | loss tokens/s: 58039.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.673638E-01 | lm-global-supervised-tokens: 1.299297E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.671735E-02 | lm-token-buffer-mib: 2.359202E+02 | global_load_balancing_loss: 1.001746E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11166.186 | params rms: 0.060461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:59:29.927573 | finish at 2026-10-17 19:10:57 + [2026-10-15 16:15:08] iteration 1565/ 2384 | consumed samples: 50080 | elapsed time per iteration (ms): 220986.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 75919.4 | valid tokens/s: 75919.4 | physical tokens/s: 76824.0 | loss tokens/s: 57018.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.013259E-01 | lm-global-supervised-tokens: 1.260034E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.938531E-02 | lm-token-buffer-mib: 2.343030E+02 | global_load_balancing_loss: 9.997008E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11166.157 | params rms: 0.060461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:16:28.106957 | finish at 2026-10-17 18:31:36 + [2026-10-15 16:18:47] iteration 1566/ 2384 | consumed samples: 50112 | elapsed time per iteration (ms): 219410.9 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 76464.8 | valid tokens/s: 76464.8 | physical tokens/s: 77067.2 | loss tokens/s: 55576.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.262597E-01 | lm-global-supervised-tokens: 1.219416E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.283648E-02 | lm-token-buffer-mib: 2.335948E+02 | global_load_balancing_loss: 9.998002E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11166.127 | params rms: 0.060461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:51:18.109079 | finish at 2026-10-17 18:10:05 + [2026-10-15 16:22:28] iteration 1567/ 2384 | consumed samples: 50144 | elapsed time per iteration (ms): 220350.0 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 76138.9 | valid tokens/s: 76138.9 | physical tokens/s: 77038.8 | loss tokens/s: 57028.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.705429E-01 | lm-global-supervised-tokens: 1.256627E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.051386E-02 | lm-token-buffer-mib: 2.325976E+02 | global_load_balancing_loss: 9.990817E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11166.097 | params rms: 0.060461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:00:25.940377 | finish at 2026-10-17 18:22:54 + [2026-10-15 16:26:11] iteration 1568/ 2384 | consumed samples: 50176 | elapsed time per iteration (ms): 223560.7 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 75045.4 | valid tokens/s: 75045.4 | physical tokens/s: 76206.6 | loss tokens/s: 60283.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.975913E-01 | lm-global-supervised-tokens: 1.347708E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.415329E-02 | lm-token-buffer-mib: 2.354988E+02 | global_load_balancing_loss: 1.000035E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11166.068 | params rms: 0.060461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:40:25.558971 | finish at 2026-10-17 19:06:37 + [2026-10-15 16:29:55] iteration 1569/ 2384 | consumed samples: 50208 | elapsed time per iteration (ms): 223987.9 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 74902.2 | valid tokens/s: 74902.2 | physical tokens/s: 75944.1 | loss tokens/s: 54717.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.553680E-01 | lm-global-supervised-tokens: 1.225614E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.423855E-02 | lm-token-buffer-mib: 2.359724E+02 | global_load_balancing_loss: 9.996910E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11166.040 | params rms: 0.060460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:42:30.130160 | finish at 2026-10-17 19:12:25 + [2026-10-15 16:34:00] iteration 1570/ 2384 | consumed samples: 50240 | elapsed time per iteration (ms): 244771.9 | throughput per GPU (TFLOP/s/GPU): 17.8 | MFU 5.72% | FLOPs estimator: workload-aware | logical tokens/s: 68542.1 | valid tokens/s: 68542.1 | physical tokens/s: 68852.9 | loss tokens/s: 52750.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.685548E-01 | lm-global-supervised-tokens: 1.291174E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.220912E-02 | lm-token-buffer-mib: 2.331408E+02 | global_load_balancing_loss: 1.001086E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11166.012 | params rms: 0.060460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:20:44.358960 | finish at 2026-10-17 23:54:44 + [2026-10-15 16:37:44] iteration 1571/ 2384 | consumed samples: 50272 | elapsed time per iteration (ms): 223683.9 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 75004.1 | valid tokens/s: 75004.1 | physical tokens/s: 75789.5 | loss tokens/s: 56927.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.426255E-01 | lm-global-supervised-tokens: 1.273381E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.606880E-02 | lm-token-buffer-mib: 2.349763E+02 | global_load_balancing_loss: 1.000838E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11165.987 | params rms: 0.060460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:30:55.035607 | finish at 2026-10-17 19:08:39 + [2026-10-15 16:41:26] iteration 1572/ 2384 | consumed samples: 50304 | elapsed time per iteration (ms): 222696.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75336.5 | valid tokens/s: 75336.5 | physical tokens/s: 75820.9 | loss tokens/s: 56600.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.422894E-01 | lm-global-supervised-tokens: 1.260469E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.932163E-02 | lm-token-buffer-mib: 2.333836E+02 | global_load_balancing_loss: 1.001583E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11165.960 | params rms: 0.060460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:13:49.747969 | finish at 2026-10-17 18:55:16 + [2026-10-15 16:45:05] iteration 1573/ 2384 | consumed samples: 50336 | elapsed time per iteration (ms): 218428.9 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.14% | FLOPs estimator: workload-aware | logical tokens/s: 76808.5 | valid tokens/s: 76808.5 | physical tokens/s: 78096.0 | loss tokens/s: 59149.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.627133E-01 | lm-global-supervised-tokens: 1.291986E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.253088E-02 | lm-token-buffer-mib: 2.328956E+02 | global_load_balancing_loss: 1.000458E+00 | loss scale: 1.0 | grad norm: 0.251 | params norm: 11165.933 | params rms: 0.060460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:12:25.851051 | finish at 2026-10-17 17:57:31 + [2026-10-15 16:48:51] iteration 1574/ 2384 | consumed samples: 50368 | elapsed time per iteration (ms): 226662.5 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 74018.4 | valid tokens/s: 74018.4 | physical tokens/s: 74933.6 | loss tokens/s: 54706.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.466240E-01 | lm-global-supervised-tokens: 1.239979E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.249125E-02 | lm-token-buffer-mib: 2.362853E+02 | global_load_balancing_loss: 1.000704E+00 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11165.904 | params rms: 0.060460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:59:56.603680 | finish at 2026-10-17 19:48:48 + [2026-10-15 16:52:33] iteration 1575/ 2384 | consumed samples: 50400 | elapsed time per iteration (ms): 221695.9 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 75676.5 | valid tokens/s: 75676.5 | physical tokens/s: 76387.4 | loss tokens/s: 56883.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.014980E-01 | lm-global-supervised-tokens: 1.261092E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.256346E-02 | lm-token-buffer-mib: 2.341684E+02 | global_load_balancing_loss: 1.000181E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11165.876 | params rms: 0.060460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:49:11.980177 | finish at 2026-10-17 18:41:45 + [2026-10-15 16:56:17] iteration 1576/ 2384 | consumed samples: 50432 | elapsed time per iteration (ms): 224151.3 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 74847.5 | valid tokens/s: 74847.5 | physical tokens/s: 75168.9 | loss tokens/s: 52500.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.436435E-01 | lm-global-supervised-tokens: 1.176801E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.448944E-02 | lm-token-buffer-mib: 2.322408E+02 | global_load_balancing_loss: 1.000570E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11165.847 | params rms: 0.060459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:18:34.241693 | finish at 2026-10-17 19:14:52 + [2026-10-15 17:00:00] iteration 1577/ 2384 | consumed samples: 50464 | elapsed time per iteration (ms): 222987.0 | throughput per GPU (TFLOP/s/GPU): 18.2 | MFU 5.82% | FLOPs estimator: workload-aware | logical tokens/s: 75238.4 | valid tokens/s: 75238.4 | physical tokens/s: 76041.6 | loss tokens/s: 56506.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.083463E-01 | lm-global-supervised-tokens: 1.260028E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.248651E-02 | lm-token-buffer-mib: 2.352969E+02 | global_load_balancing_loss: 1.000003E+00 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11165.817 | params rms: 0.060459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:59:10.480323 | finish at 2026-10-17 18:59:11 + [2026-10-15 17:03:39] iteration 1578/ 2384 | consumed samples: 50496 | elapsed time per iteration (ms): 219032.8 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 76596.7 | valid tokens/s: 76596.7 | physical tokens/s: 76870.3 | loss tokens/s: 56126.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.408251E-01 | lm-global-supervised-tokens: 1.229355E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.176261E-02 | lm-token-buffer-mib: 2.334162E+02 | global_load_balancing_loss: 9.996058E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11165.790 | params rms: 0.060459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:02:20.450795 | finish at 2026-10-17 18:06:00 + [2026-10-15 17:07:24] iteration 1579/ 2384 | consumed samples: 50528 | elapsed time per iteration (ms): 224880.8 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 74604.8 | valid tokens/s: 74604.8 | physical tokens/s: 75735.6 | loss tokens/s: 53739.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.472960E-01 | lm-global-supervised-tokens: 1.208501E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.925015E-02 | lm-token-buffer-mib: 2.341057E+02 | global_load_balancing_loss: 1.000189E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11165.760 | params rms: 0.060459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:17:09.035946 | finish at 2026-10-17 19:24:33 + [2026-10-15 17:11:27] iteration 1580/ 2384 | consumed samples: 50560 | elapsed time per iteration (ms): 242970.2 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 69050.4 | valid tokens/s: 69050.4 | physical tokens/s: 69679.3 | loss tokens/s: 54590.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.026244E-01 | lm-global-supervised-tokens: 1.326393E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.244761E-02 | lm-token-buffer-mib: 2.332308E+02 | global_load_balancing_loss: 9.995700E-01 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11165.729 | params rms: 0.060459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:15:48.061360 | finish at 2026-10-17 23:27:15 + [2026-10-15 17:15:08] iteration 1581/ 2384 | consumed samples: 50592 | elapsed time per iteration (ms): 220610.8 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 76048.8 | valid tokens/s: 76048.8 | physical tokens/s: 76506.4 | loss tokens/s: 56780.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.600031E-01 | lm-global-supervised-tokens: 1.252638E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.493091E-02 | lm-token-buffer-mib: 2.328573E+02 | global_load_balancing_loss: 1.000193E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11165.698 | params rms: 0.060459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:12:30.486207 | finish at 2026-10-17 18:27:38 + [2026-10-15 17:18:49] iteration 1582/ 2384 | consumed samples: 50624 | elapsed time per iteration (ms): 221445.1 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75762.3 | valid tokens/s: 75762.3 | physical tokens/s: 76124.5 | loss tokens/s: 54031.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.152682E-01 | lm-global-supervised-tokens: 1.196509E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.667814E-02 | lm-token-buffer-mib: 2.333215E+02 | global_load_balancing_loss: 1.001034E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11165.667 | params rms: 0.060458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:19:58.984214 | finish at 2026-10-17 18:38:48 + [2026-10-15 17:22:35] iteration 1583/ 2384 | consumed samples: 50656 | elapsed time per iteration (ms): 225519.5 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 74393.5 | valid tokens/s: 74393.5 | physical tokens/s: 75629.8 | loss tokens/s: 57482.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.457957E-01 | lm-global-supervised-tokens: 1.296332E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.092774E-02 | lm-token-buffer-mib: 2.361875E+02 | global_load_balancing_loss: 1.000539E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11165.633 | params rms: 0.060458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:10:41.100225 | finish at 2026-10-17 19:33:16 + [2026-10-15 17:26:20] iteration 1584/ 2384 | consumed samples: 50688 | elapsed time per iteration (ms): 225458.6 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 74413.7 | valid tokens/s: 74413.7 | physical tokens/s: 75488.2 | loss tokens/s: 57755.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.621272E-01 | lm-global-supervised-tokens: 1.302138E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.873605E-02 | lm-token-buffer-mib: 2.327695E+02 | global_load_balancing_loss: 1.000466E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11165.600 | params rms: 0.060458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:06:06.871262 | finish at 2026-10-17 19:32:27 + [2026-10-15 17:30:00] iteration 1585/ 2384 | consumed samples: 50720 | elapsed time per iteration (ms): 220238.8 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 76177.3 | valid tokens/s: 76177.3 | physical tokens/s: 76769.0 | loss tokens/s: 59970.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.842898E-01 | lm-global-supervised-tokens: 1.320792E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.177776E-02 | lm-token-buffer-mib: 2.334052E+02 | global_load_balancing_loss: 1.001072E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11165.564 | params rms: 0.060458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:52:50.808478 | finish at 2026-10-17 18:22:51 + [2026-10-15 17:33:46] iteration 1586/ 2384 | consumed samples: 50752 | elapsed time per iteration (ms): 225160.3 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 74512.3 | valid tokens/s: 74512.3 | physical tokens/s: 75628.3 | loss tokens/s: 54288.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.237940E-01 | lm-global-supervised-tokens: 1.222364E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.014279E-02 | lm-token-buffer-mib: 2.339729E+02 | global_load_balancing_loss: 1.001660E+00 | loss scale: 1.0 | grad norm: 0.263 | params norm: 11165.530 | params rms: 0.060458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:54:37.884976 | finish at 2026-10-17 19:28:24 + [2026-10-15 17:37:30] iteration 1587/ 2384 | consumed samples: 50784 | elapsed time per iteration (ms): 224390.3 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 74767.9 | valid tokens/s: 74767.9 | physical tokens/s: 75493.3 | loss tokens/s: 58818.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.908926E-01 | lm-global-supervised-tokens: 1.319826E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.476365E-02 | lm-token-buffer-mib: 2.349423E+02 | global_load_balancing_loss: 9.986909E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11165.497 | params rms: 0.060458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:40:39.043856 | finish at 2026-10-17 19:18:09 + [2026-10-15 17:41:17] iteration 1588/ 2384 | consumed samples: 50816 | elapsed time per iteration (ms): 227438.5 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 73765.8 | valid tokens/s: 73765.8 | physical tokens/s: 74346.2 | loss tokens/s: 57880.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.784367E-01 | lm-global-supervised-tokens: 1.316427E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.237570E-02 | lm-token-buffer-mib: 2.344983E+02 | global_load_balancing_loss: 9.997101E-01 | loss scale: 1.0 | grad norm: 0.308 | params norm: 11165.463 | params rms: 0.060457 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:17:21.006847 | finish at 2026-10-17 19:58:38 + [2026-10-15 17:45:52] iteration 1589/ 2384 | consumed samples: 50848 | elapsed time per iteration (ms): 274510.2 | throughput per GPU (TFLOP/s/GPU): 18.2 | MFU 5.84% | FLOPs estimator: workload-aware | logical tokens/s: 61116.8 | valid tokens/s: 61116.8 | physical tokens/s: 62811.2 | loss tokens/s: 45099.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.639816E-01 | lm-global-supervised-tokens: 1.238019E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.155230E-02 | lm-token-buffer-mib: 2.432141E+02 | global_load_balancing_loss: 9.993586E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11165.429 | params rms: 0.060457 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:37:15.623235 | finish at 2026-10-18 06:23:08 + [2026-10-15 17:49:56] iteration 1590/ 2384 | consumed samples: 50880 | elapsed time per iteration (ms): 244264.9 | throughput per GPU (TFLOP/s/GPU): 18.0 | MFU 5.78% | FLOPs estimator: workload-aware | logical tokens/s: 68684.5 | valid tokens/s: 68684.5 | physical tokens/s: 69160.6 | loss tokens/s: 52870.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.168607E-01 | lm-global-supervised-tokens: 1.291432E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.639356E-02 | lm-token-buffer-mib: 2.332891E+02 | global_load_balancing_loss: 1.000388E+00 | loss scale: 1.0 | grad norm: 0.550 | params norm: 11165.398 | params rms: 0.060457 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:52:26.327736 | finish at 2026-10-17 23:42:23 + [2026-10-15 17:53:36] iteration 1591/ 2384 | consumed samples: 50912 | elapsed time per iteration (ms): 219288.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 76507.6 | valid tokens/s: 76507.5 | physical tokens/s: 77184.1 | loss tokens/s: 59326.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.647177E-01 | lm-global-supervised-tokens: 1.300966E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.459299E-02 | lm-token-buffer-mib: 2.342353E+02 | global_load_balancing_loss: 9.997951E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11165.367 | params rms: 0.060457 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:18:15.501495 | finish at 2026-10-17 18:11:51 + [2026-10-15 17:57:16] iteration 1592/ 2384 | consumed samples: 50944 | elapsed time per iteration (ms): 220428.2 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 76111.8 | valid tokens/s: 76111.8 | physical tokens/s: 76855.8 | loss tokens/s: 59604.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.633781E-01 | lm-global-supervised-tokens: 1.313841E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.713162E-02 | lm-token-buffer-mib: 2.344541E+02 | global_load_balancing_loss: 1.001594E+00 | loss scale: 1.0 | grad norm: 0.212 | params norm: 11165.333 | params rms: 0.060457 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:29:39.109858 | finish at 2026-10-17 18:26:55 + [2026-10-15 18:00:56] iteration 1593/ 2384 | consumed samples: 50976 | elapsed time per iteration (ms): 219699.2 | throughput per GPU (TFLOP/s/GPU): 18.2 | MFU 5.82% | FLOPs estimator: workload-aware | logical tokens/s: 76364.4 | valid tokens/s: 76364.4 | physical tokens/s: 76735.9 | loss tokens/s: 53644.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.895811E-01 | lm-global-supervised-tokens: 1.178571E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.813805E-02 | lm-token-buffer-mib: 2.327834E+02 | global_load_balancing_loss: 1.001131E+00 | loss scale: 1.0 | grad norm: 0.217 | params norm: 11165.297 | params rms: 0.060456 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:16:22.082220 | finish at 2026-10-17 18:17:18 + [2026-10-15 18:04:36] iteration 1594/ 2384 | consumed samples: 51008 | elapsed time per iteration (ms): 220305.0 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 76154.4 | valid tokens/s: 76154.4 | physical tokens/s: 76392.1 | loss tokens/s: 57367.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.196276E-01 | lm-global-supervised-tokens: 1.263839E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.601098E-02 | lm-token-buffer-mib: 2.328687E+02 | global_load_balancing_loss: 1.001791E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11165.261 | params rms: 0.060456 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:20:40.972655 | finish at 2026-10-17 18:25:17 + [2026-10-15 18:08:18] iteration 1595/ 2384 | consumed samples: 51040 | elapsed time per iteration (ms): 222156.6 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 75519.4 | valid tokens/s: 75519.4 | physical tokens/s: 77236.2 | loss tokens/s: 58940.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.067629E-01 | lm-global-supervised-tokens: 1.309399E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.686099E-02 | lm-token-buffer-mib: 2.365189E+02 | global_load_balancing_loss: 1.000317E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11165.225 | params rms: 0.060456 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:41:21.571319 | finish at 2026-10-17 18:49:40 + [2026-10-15 18:12:03] iteration 1596/ 2384 | consumed samples: 51072 | elapsed time per iteration (ms): 225180.8 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.88% | FLOPs estimator: workload-aware | logical tokens/s: 74505.4 | valid tokens/s: 74505.4 | physical tokens/s: 74995.8 | loss tokens/s: 56694.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.261151E-01 | lm-global-supervised-tokens: 1.276643E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.226979E-02 | lm-token-buffer-mib: 2.323839E+02 | global_load_balancing_loss: 9.992103E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11165.189 | params rms: 0.060456 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:17:22.488593 | finish at 2026-10-17 19:29:26 + [2026-10-15 18:15:45] iteration 1597/ 2384 | consumed samples: 51104 | elapsed time per iteration (ms): 221395.3 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 75779.3 | valid tokens/s: 75779.2 | physical tokens/s: 76330.7 | loss tokens/s: 59384.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.852144E-01 | lm-global-supervised-tokens: 1.314733E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.279142E-02 | lm-token-buffer-mib: 2.339742E+02 | global_load_balancing_loss: 9.989921E-01 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11165.155 | params rms: 0.060456 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:23:58.119043 | finish at 2026-10-17 18:39:43 + [2026-10-15 18:19:30] iteration 1598/ 2384 | consumed samples: 51136 | elapsed time per iteration (ms): 224949.1 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 74582.1 | valid tokens/s: 74582.1 | physical tokens/s: 75168.1 | loss tokens/s: 55417.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.432132E-01 | lm-global-supervised-tokens: 1.246613E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.421487E-02 | lm-token-buffer-mib: 2.344524E+02 | global_load_balancing_loss: 9.996157E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11165.119 | params rms: 0.060455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:06:49.953635 | finish at 2026-10-17 19:26:20 + [2026-10-15 18:23:15] iteration 1599/ 2384 | consumed samples: 51168 | elapsed time per iteration (ms): 225372.6 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 74442.0 | valid tokens/s: 74442.0 | physical tokens/s: 75005.0 | loss tokens/s: 55024.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.733060E-01 | lm-global-supervised-tokens: 1.240108E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.651550E-02 | lm-token-buffer-mib: 2.328157E+02 | global_load_balancing_loss: 1.000377E+00 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11165.081 | params rms: 0.060455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:08:37.493308 | finish at 2026-10-17 19:31:53 + [2026-10-15 18:27:18] iteration 1600/ 2384 | consumed samples: 51200 | elapsed time per iteration (ms): 242530.1 | throughput per GPU (TFLOP/s/GPU): 17.7 | MFU 5.66% | FLOPs estimator: workload-aware | logical tokens/s: 69175.7 | valid tokens/s: 69175.7 | physical tokens/s: 69991.8 | loss tokens/s: 53836.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.655195E-01 | lm-global-supervised-tokens: 1.305686E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.394689E-02 | lm-token-buffer-mib: 2.329082E+02 | global_load_balancing_loss: 1.000355E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11165.041 | params rms: 0.060455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:49:03.571754 | finish at 2026-10-17 23:16:21 +times across ranks (ms): + save-checkpoint: + rank 0: 24702.74 + rank 1: 24702.74 + rank 2: 24702.80 + rank 3: 24702.76 + rank 4: 24702.73 + rank 5: 24702.76 + rank 6: 24702.76 + rank 7: 24702.77 + rank 8: 24703.34 + rank 9: 24703.35 + rank 10: 24703.35 + rank 11: 24703.35 + rank 12: 24703.33 + rank 13: 24703.51 + rank 14: 24703.31 + rank 15: 24703.33 + rank 16: 24702.82 + rank 17: 24702.81 + rank 18: 24702.81 + rank 19: 24702.81 + rank 20: 24702.79 + rank 21: 24702.80 + rank 22: 24702.82 + rank 23: 24702.82 + rank 24: 24703.05 + rank 25: 24703.07 + rank 26: 24703.10 + rank 27: 24703.02 + rank 28: 24703.04 + rank 29: 24703.07 + rank 30: 24703.04 + rank 31: 24703.35 + rank 32: 24702.92 + rank 33: 24702.89 + rank 34: 24702.92 + rank 35: 24702.97 + rank 36: 24702.82 + rank 37: 24702.86 + rank 38: 24702.79 + rank 39: 24702.81 + rank 40: 24702.86 + rank 41: 24702.84 + rank 42: 24702.85 + rank 43: 24702.84 + rank 44: 24702.83 + rank 45: 24704.12 + rank 46: 24702.82 + rank 47: 24702.82 + rank 48: 24702.74 + rank 49: 24702.89 + rank 50: 24702.93 + rank 51: 24702.86 + rank 52: 24702.94 + rank 53: 24703.12 + rank 54: 24702.89 + rank 55: 24702.86 + rank 56: 24702.68 + rank 57: 24702.70 + rank 58: 24702.70 + rank 59: 24702.71 + rank 60: 24702.71 + rank 61: 24702.72 + rank 62: 24702.68 + rank 63: 24702.67 + [2026-10-15 18:31:22] iteration 1601/ 2384 | consumed samples: 51232 | elapsed time per iteration (ms): 219676.3 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.79% | FLOPs estimator: workload-aware | logical tokens/s: 76372.3 | valid tokens/s: 76372.3 | physical tokens/s: 78042.1 | loss tokens/s: 58876.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.436396E-01 | lm-global-supervised-tokens: 1.293377E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.654293E-02 | lm-token-buffer-mib: 2.348040E+02 | global_load_balancing_loss: 1.000539E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11165.005 | params rms: 0.060455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:46:46.546859 | finish at 2026-10-17 18:18:09 + [2026-10-15 18:35:10] iteration 1602/ 2384 | consumed samples: 51264 | elapsed time per iteration (ms): 227715.4 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.17% | FLOPs estimator: workload-aware | logical tokens/s: 73676.2 | valid tokens/s: 73676.2 | physical tokens/s: 75650.9 | loss tokens/s: 57087.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.221460E-01 | lm-global-supervised-tokens: 1.299977E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.617974E-02 | lm-token-buffer-mib: 2.372374E+02 | global_load_balancing_loss: 1.001463E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11164.978 | params rms: 0.060455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:27:53.434954 | finish at 2026-10-17 20:03:03 + [2026-10-15 18:38:52] iteration 1603/ 2384 | consumed samples: 51296 | elapsed time per iteration (ms): 222028.5 | throughput per GPU (TFLOP/s/GPU): 26.4 | MFU 8.45% | FLOPs estimator: workload-aware | logical tokens/s: 75563.2 | valid tokens/s: 75563.2 | physical tokens/s: 76015.9 | loss tokens/s: 59695.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.588183E-01 | lm-global-supervised-tokens: 1.325403E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.704705E-02 | lm-token-buffer-mib: 2.336117E+02 | global_load_balancing_loss: 1.000500E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11164.951 | params rms: 0.060455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:10:04.274762 | finish at 2026-10-17 18:48:56 + [2026-10-15 18:42:35] iteration 1604/ 2384 | consumed samples: 51328 | elapsed time per iteration (ms): 222875.5 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 75276.0 | valid tokens/s: 75276.0 | physical tokens/s: 75714.1 | loss tokens/s: 51591.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.082049E-01 | lm-global-supervised-tokens: 1.149857E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.691614E-02 | lm-token-buffer-mib: 2.334204E+02 | global_load_balancing_loss: 1.000977E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11164.924 | params rms: 0.060454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:17:22.918797 | finish at 2026-10-17 18:59:58 + [2026-10-15 18:46:15] iteration 1605/ 2384 | consumed samples: 51360 | elapsed time per iteration (ms): 220043.2 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 76244.8 | valid tokens/s: 76244.8 | physical tokens/s: 77128.1 | loss tokens/s: 57989.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.055983E-01 | lm-global-supervised-tokens: 1.276020E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.299143E-02 | lm-token-buffer-mib: 2.344561E+02 | global_load_balancing_loss: 1.000522E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11164.900 | params rms: 0.060454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:36:53.667114 | finish at 2026-10-17 18:23:08 + [2026-10-15 18:49:59] iteration 1606/ 2384 | consumed samples: 51392 | elapsed time per iteration (ms): 224758.7 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 74645.3 | valid tokens/s: 74645.3 | physical tokens/s: 75279.0 | loss tokens/s: 59488.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.910030E-01 | lm-global-supervised-tokens: 1.337058E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.895346E-02 | lm-token-buffer-mib: 2.349564E+02 | global_load_balancing_loss: 9.997323E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11164.876 | params rms: 0.060454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:34:22.255162 | finish at 2026-10-17 19:24:22 + [2026-10-15 18:53:41] iteration 1607/ 2384 | consumed samples: 51424 | elapsed time per iteration (ms): 221156.4 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 75861.1 | valid tokens/s: 75861.1 | physical tokens/s: 76273.4 | loss tokens/s: 54085.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.852236E-01 | lm-global-supervised-tokens: 1.196130E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.372855E-02 | lm-token-buffer-mib: 2.331053E+02 | global_load_balancing_loss: 9.998047E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11164.852 | params rms: 0.060454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:43:58.528145 | finish at 2026-10-17 18:37:39 + [2026-10-15 18:57:23] iteration 1608/ 2384 | consumed samples: 51456 | elapsed time per iteration (ms): 222218.3 | throughput per GPU (TFLOP/s/GPU): 18.5 | MFU 5.93% | FLOPs estimator: workload-aware | logical tokens/s: 75498.8 | valid tokens/s: 75498.8 | physical tokens/s: 76417.5 | loss tokens/s: 58428.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.240119E-01 | lm-global-supervised-tokens: 1.298384E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.256941E-02 | lm-token-buffer-mib: 2.360428E+02 | global_load_balancing_loss: 1.000301E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11164.828 | params rms: 0.060454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:54:01.396441 | finish at 2026-10-17 18:51:24 + [2026-10-15 19:01:04] iteration 1609/ 2384 | consumed samples: 51488 | elapsed time per iteration (ms): 221331.0 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 75801.1 | valid tokens/s: 75801.1 | physical tokens/s: 76564.9 | loss tokens/s: 56171.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.217862E-01 | lm-global-supervised-tokens: 1.243251E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.538210E-02 | lm-token-buffer-mib: 2.339836E+02 | global_load_balancing_loss: 1.000048E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11164.801 | params rms: 0.060454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:38:51.513798 | finish at 2026-10-17 18:39:56 + [2026-10-15 19:05:10] iteration 1610/ 2384 | consumed samples: 51520 | elapsed time per iteration (ms): 246273.9 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 68124.2 | valid tokens/s: 68124.2 | physical tokens/s: 68709.2 | loss tokens/s: 49521.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.831284E-01 | lm-global-supervised-tokens: 1.219578E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.397607E-02 | lm-token-buffer-mib: 2.344455E+02 | global_load_balancing_loss: 1.000113E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11164.774 | params rms: 0.060454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:56:55.991797 | finish at 2026-10-18 00:02:06 + [2026-10-15 19:08:54] iteration 1611/ 2384 | consumed samples: 51552 | elapsed time per iteration (ms): 223140.4 | throughput per GPU (TFLOP/s/GPU): 26.9 | MFU 8.62% | FLOPs estimator: workload-aware | logical tokens/s: 75186.7 | valid tokens/s: 75186.7 | physical tokens/s: 75779.4 | loss tokens/s: 52262.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.272648E-01 | lm-global-supervised-tokens: 1.166178E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.267123E-02 | lm-token-buffer-mib: 2.337256E+02 | global_load_balancing_loss: 1.000676E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11164.748 | params rms: 0.060453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:54:47.533940 | finish at 2026-10-17 19:03:41 + [2026-10-15 19:12:38] iteration 1612/ 2384 | consumed samples: 51584 | elapsed time per iteration (ms): 223972.2 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.08% | FLOPs estimator: workload-aware | logical tokens/s: 74907.3 | valid tokens/s: 74907.3 | physical tokens/s: 75943.1 | loss tokens/s: 57318.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.755521E-01 | lm-global-supervised-tokens: 1.283783E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.704232E-02 | lm-token-buffer-mib: 2.336143E+02 | global_load_balancing_loss: 9.999734E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11164.723 | params rms: 0.060453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:01:46.548091 | finish at 2026-10-17 19:14:24 + [2026-10-15 19:16:20] iteration 1613/ 2384 | consumed samples: 51616 | elapsed time per iteration (ms): 222795.8 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 75303.0 | valid tokens/s: 75303.0 | physical tokens/s: 75762.4 | loss tokens/s: 58968.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.742039E-01 | lm-global-supervised-tokens: 1.313797E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.417275E-02 | lm-token-buffer-mib: 2.323769E+02 | global_load_balancing_loss: 1.001451E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11164.701 | params rms: 0.060453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:42:55.577954 | finish at 2026-10-17 18:59:16 + [2026-10-15 19:20:03] iteration 1614/ 2384 | consumed samples: 51648 | elapsed time per iteration (ms): 222273.0 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 75480.2 | valid tokens/s: 75480.2 | physical tokens/s: 76158.9 | loss tokens/s: 58347.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.452689E-01 | lm-global-supervised-tokens: 1.296898E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.265784E-02 | lm-token-buffer-mib: 2.341486E+02 | global_load_balancing_loss: 1.000204E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11164.677 | params rms: 0.060453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:32:30.181363 | finish at 2026-10-17 18:52:33 + [2026-10-15 19:23:45] iteration 1615/ 2384 | consumed samples: 51680 | elapsed time per iteration (ms): 222367.3 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75448.1 | valid tokens/s: 75448.1 | physical tokens/s: 75965.4 | loss tokens/s: 54989.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.251539E-01 | lm-global-supervised-tokens: 1.222792E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.508265E-02 | lm-token-buffer-mib: 2.332177E+02 | global_load_balancing_loss: 1.000297E+00 | loss scale: 1.0 | grad norm: 0.248 | params norm: 11164.653 | params rms: 0.060453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:30:00.483978 | finish at 2026-10-17 18:53:45 + [2026-10-15 19:27:29] iteration 1616/ 2384 | consumed samples: 51712 | elapsed time per iteration (ms): 223590.8 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75035.3 | valid tokens/s: 75035.3 | physical tokens/s: 76256.7 | loss tokens/s: 60832.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.132140E-01 | lm-global-supervised-tokens: 1.360168E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.756614E-02 | lm-token-buffer-mib: 2.333828E+02 | global_load_balancing_loss: 9.999304E-01 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11164.630 | params rms: 0.060453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:41:57.767395 | finish at 2026-10-17 19:09:26 + [2026-10-15 19:31:12] iteration 1617/ 2384 | consumed samples: 51744 | elapsed time per iteration (ms): 223752.2 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.28% | FLOPs estimator: workload-aware | logical tokens/s: 74981.2 | valid tokens/s: 74981.2 | physical tokens/s: 75990.3 | loss tokens/s: 58365.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.643251E-01 | lm-global-supervised-tokens: 1.305933E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.253226E-02 | lm-token-buffer-mib: 2.356633E+02 | global_load_balancing_loss: 1.000169E+00 | loss scale: 1.0 | grad norm: 0.355 | params norm: 11164.609 | params rms: 0.060453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:40:17.968402 | finish at 2026-10-17 19:11:30 + [2026-10-15 19:34:59] iteration 1618/ 2384 | consumed samples: 51776 | elapsed time per iteration (ms): 226752.2 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.64% | FLOPs estimator: workload-aware | logical tokens/s: 73989.1 | valid tokens/s: 73989.1 | physical tokens/s: 75273.1 | loss tokens/s: 53657.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.770237E-01 | lm-global-supervised-tokens: 1.216693E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.742201E-02 | lm-token-buffer-mib: 2.335711E+02 | global_load_balancing_loss: 9.997028E-01 | loss scale: 1.0 | grad norm: 0.193 | params norm: 11164.588 | params rms: 0.060453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:14:52.222736 | finish at 2026-10-17 19:49:51 + [2026-10-15 19:38:42] iteration 1619/ 2384 | consumed samples: 51808 | elapsed time per iteration (ms): 222865.0 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 75279.5 | valid tokens/s: 75279.5 | physical tokens/s: 75823.3 | loss tokens/s: 58741.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.034857E-01 | lm-global-supervised-tokens: 1.309149E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.035521E-02 | lm-token-buffer-mib: 2.331957E+02 | global_load_balancing_loss: 1.000081E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11164.564 | params rms: 0.060452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:21:31.708592 | finish at 2026-10-17 19:00:14 + [2026-10-15 19:42:46] iteration 1620/ 2384 | consumed samples: 51840 | elapsed time per iteration (ms): 244269.7 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.14% | FLOPs estimator: workload-aware | logical tokens/s: 68683.1 | valid tokens/s: 68683.1 | physical tokens/s: 69577.4 | loss tokens/s: 52676.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.475478E-01 | lm-global-supervised-tokens: 1.286736E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.315632E-02 | lm-token-buffer-mib: 2.346102E+02 | global_load_balancing_loss: 1.000689E+00 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11164.538 | params rms: 0.060452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:50:22.071968 | finish at 2026-10-17 23:33:08 + [2026-10-15 19:46:30] iteration 1621/ 2384 | consumed samples: 51872 | elapsed time per iteration (ms): 223967.1 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 74909.1 | valid tokens/s: 74909.1 | physical tokens/s: 75686.0 | loss tokens/s: 56874.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.573409E-01 | lm-global-supervised-tokens: 1.273810E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.415423E-02 | lm-token-buffer-mib: 2.365386E+02 | global_load_balancing_loss: 1.001145E+00 | loss scale: 1.0 | grad norm: 0.313 | params norm: 11164.513 | params rms: 0.060452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:28:06.884130 | finish at 2026-10-17 19:14:37 + [2026-10-15 19:50:13] iteration 1622/ 2384 | consumed samples: 51904 | elapsed time per iteration (ms): 222706.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75332.9 | valid tokens/s: 75332.9 | physical tokens/s: 76452.0 | loss tokens/s: 55826.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.521492E-01 | lm-global-supervised-tokens: 1.243284E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.819354E-02 | lm-token-buffer-mib: 2.328332E+02 | global_load_balancing_loss: 1.000426E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11164.484 | params rms: 0.060452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:08:22.522180 | finish at 2026-10-17 18:58:35 + [2026-10-15 19:53:53] iteration 1623/ 2384 | consumed samples: 51936 | elapsed time per iteration (ms): 220125.5 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 76216.4 | valid tokens/s: 76216.4 | physical tokens/s: 76886.2 | loss tokens/s: 55147.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.287605E-01 | lm-global-supervised-tokens: 1.213938E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.359091E-02 | lm-token-buffer-mib: 2.334062E+02 | global_load_balancing_loss: 1.000485E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11164.456 | params rms: 0.060452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:31:55.507831 | finish at 2026-10-17 18:25:49 + [2026-10-15 19:57:35] iteration 1624/ 2384 | consumed samples: 51968 | elapsed time per iteration (ms): 222398.0 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 75437.8 | valid tokens/s: 75437.8 | physical tokens/s: 75956.7 | loss tokens/s: 56943.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.193594E-01 | lm-global-supervised-tokens: 1.266406E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.493498E-02 | lm-token-buffer-mib: 2.338419E+02 | global_load_balancing_loss: 1.000257E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11164.429 | params rms: 0.060452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:57:02.470942 | finish at 2026-10-17 18:54:38 + [2026-10-15 20:01:17] iteration 1625/ 2384 | consumed samples: 52000 | elapsed time per iteration (ms): 221737.4 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 75662.3 | valid tokens/s: 75662.3 | physical tokens/s: 76023.6 | loss tokens/s: 61789.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.534713E-01 | lm-global-supervised-tokens: 1.370106E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.540727E-02 | lm-token-buffer-mib: 2.332488E+02 | global_load_balancing_loss: 1.000670E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11164.401 | params rms: 0.060452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:44:58.719576 | finish at 2026-10-17 18:46:16 + [2026-10-15 20:05:04] iteration 1626/ 2384 | consumed samples: 52032 | elapsed time per iteration (ms): 227061.4 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 73888.2 | valid tokens/s: 73888.2 | physical tokens/s: 74865.4 | loss tokens/s: 53033.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.752139E-01 | lm-global-supervised-tokens: 1.204189E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.164640E-02 | lm-token-buffer-mib: 2.377477E+02 | global_load_balancing_loss: 9.992545E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11164.369 | params rms: 0.060451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:48:32.565730 | finish at 2026-10-17 19:53:37 + [2026-10-15 20:08:50] iteration 1627/ 2384 | consumed samples: 52064 | elapsed time per iteration (ms): 226299.6 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.05% | FLOPs estimator: workload-aware | logical tokens/s: 74137.1 | valid tokens/s: 74137.0 | physical tokens/s: 74558.9 | loss tokens/s: 58161.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.118980E-01 | lm-global-supervised-tokens: 1.316197E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.602324E-02 | lm-token-buffer-mib: 2.339877E+02 | global_load_balancing_loss: 1.000084E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11164.335 | params rms: 0.060451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:35:08.814440 | finish at 2026-10-17 19:43:59 + [2026-10-15 20:12:37] iteration 1628/ 2384 | consumed samples: 52096 | elapsed time per iteration (ms): 226806.5 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 73971.4 | valid tokens/s: 73971.4 | physical tokens/s: 75127.0 | loss tokens/s: 54427.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.027712E-01 | lm-global-supervised-tokens: 1.234461E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.667064E-02 | lm-token-buffer-mib: 2.382021E+02 | global_load_balancing_loss: 1.001033E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11164.301 | params rms: 0.060451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:37:45.709462 | finish at 2026-10-17 19:50:23 + [2026-10-15 20:16:18] iteration 1629/ 2384 | consumed samples: 52128 | elapsed time per iteration (ms): 221072.7 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 75890.0 | valid tokens/s: 75890.0 | physical tokens/s: 76653.2 | loss tokens/s: 56869.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.402995E-01 | lm-global-supervised-tokens: 1.257219E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.256240E-02 | lm-token-buffer-mib: 2.340138E+02 | global_load_balancing_loss: 1.000930E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11164.268 | params rms: 0.060451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:21:49.889958 | finish at 2026-10-17 18:38:08 + [2026-10-15 20:20:28] iteration 1630/ 2384 | consumed samples: 52160 | elapsed time per iteration (ms): 249805.2 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 67161.0 | valid tokens/s: 67161.0 | physical tokens/s: 68002.0 | loss tokens/s: 49333.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.131220E-01 | lm-global-supervised-tokens: 1.232381E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.528556E-02 | lm-token-buffer-mib: 2.355140E+02 | global_load_balancing_loss: 9.996846E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11164.236 | params rms: 0.060451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:19:13.142807 | finish at 2026-10-18 00:39:41 + [2026-10-15 20:24:10] iteration 1631/ 2384 | consumed samples: 52192 | elapsed time per iteration (ms): 221844.4 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 75626.0 | valid tokens/s: 75626.0 | physical tokens/s: 76383.0 | loss tokens/s: 58845.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.744581E-01 | lm-global-supervised-tokens: 1.305456E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.738232E-02 | lm-token-buffer-mib: 2.336329E+02 | global_load_balancing_loss: 9.994727E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11164.203 | params rms: 0.060451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:24:08.849663 | finish at 2026-10-17 18:48:19 + [2026-10-15 20:27:55] iteration 1632/ 2384 | consumed samples: 52224 | elapsed time per iteration (ms): 225073.3 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 74541.0 | valid tokens/s: 74541.0 | physical tokens/s: 75818.7 | loss tokens/s: 56859.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.263987E-01 | lm-global-supervised-tokens: 1.279758E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.189999E-02 | lm-token-buffer-mib: 2.359135E+02 | global_load_balancing_loss: 1.000342E+00 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11164.167 | params rms: 0.060450 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:00:55.156834 | finish at 2026-10-17 19:28:50 + [2026-10-15 20:31:37] iteration 1633/ 2384 | consumed samples: 52256 | elapsed time per iteration (ms): 221495.8 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.45% | FLOPs estimator: workload-aware | logical tokens/s: 75745.0 | valid tokens/s: 75745.0 | physical tokens/s: 76310.2 | loss tokens/s: 55444.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.667064E-01 | lm-global-supervised-tokens: 1.228074E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.644848E-02 | lm-token-buffer-mib: 2.342635E+02 | global_load_balancing_loss: 1.001048E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11164.135 | params rms: 0.060450 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:12:23.376074 | finish at 2026-10-17 18:44:00 + [2026-10-15 20:35:20] iteration 1634/ 2384 | consumed samples: 52288 | elapsed time per iteration (ms): 223766.9 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.49% | FLOPs estimator: workload-aware | logical tokens/s: 74976.1 | valid tokens/s: 74976.1 | physical tokens/s: 75846.6 | loss tokens/s: 55784.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.253669E-01 | lm-global-supervised-tokens: 1.248266E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.713137E-02 | lm-token-buffer-mib: 2.349532E+02 | global_load_balancing_loss: 1.000606E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11164.102 | params rms: 0.060450 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:37:05.189531 | finish at 2026-10-17 19:12:26 + [2026-10-15 20:39:03] iteration 1635/ 2384 | consumed samples: 52320 | elapsed time per iteration (ms): 222427.7 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 75427.6 | valid tokens/s: 75427.6 | physical tokens/s: 76568.6 | loss tokens/s: 59041.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.859859E-01 | lm-global-supervised-tokens: 1.313248E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.504752E-02 | lm-token-buffer-mib: 2.356234E+02 | global_load_balancing_loss: 9.993528E-01 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11164.068 | params rms: 0.060450 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:16:38.330546 | finish at 2026-10-17 18:55:41 + [2026-10-15 20:42:46] iteration 1636/ 2384 | consumed samples: 52352 | elapsed time per iteration (ms): 223693.4 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 75000.8 | valid tokens/s: 75000.8 | physical tokens/s: 75826.6 | loss tokens/s: 54828.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.465418E-01 | lm-global-supervised-tokens: 1.226482E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.323633E-02 | lm-token-buffer-mib: 2.350280E+02 | global_load_balancing_loss: 1.001405E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11164.036 | params rms: 0.060450 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:28:42.667588 | finish at 2026-10-17 19:11:29 + [2026-10-15 20:46:27] iteration 1637/ 2384 | consumed samples: 52384 | elapsed time per iteration (ms): 220479.2 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 76094.2 | valid tokens/s: 76094.2 | physical tokens/s: 76760.2 | loss tokens/s: 53230.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.969100E-01 | lm-global-supervised-tokens: 1.173620E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.731265E-02 | lm-token-buffer-mib: 2.352816E+02 | global_load_balancing_loss: 1.000166E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11164.006 | params rms: 0.060449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:44:57.953944 | finish at 2026-10-17 18:31:25 + [2026-10-15 20:50:09] iteration 1638/ 2384 | consumed samples: 52416 | elapsed time per iteration (ms): 222241.3 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75490.9 | valid tokens/s: 75490.9 | physical tokens/s: 76006.8 | loss tokens/s: 54574.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.449643E-01 | lm-global-supervised-tokens: 1.212876E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.130559E-02 | lm-token-buffer-mib: 2.333349E+02 | global_load_balancing_loss: 9.997108E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11163.973 | params rms: 0.060449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:03:11.981955 | finish at 2026-10-17 18:53:21 + [2026-10-15 20:53:51] iteration 1639/ 2384 | consumed samples: 52448 | elapsed time per iteration (ms): 222281.2 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.42% | FLOPs estimator: workload-aware | logical tokens/s: 75477.3 | valid tokens/s: 75477.3 | physical tokens/s: 75866.4 | loss tokens/s: 56573.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.235597E-01 | lm-global-supervised-tokens: 1.257517E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.310549E-02 | lm-token-buffer-mib: 2.333214E+02 | global_load_balancing_loss: 9.999558E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11163.938 | params rms: 0.060449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:59:59.473523 | finish at 2026-10-17 18:53:51 + [2026-10-15 20:57:56] iteration 1640/ 2384 | consumed samples: 52480 | elapsed time per iteration (ms): 244139.6 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.04% | FLOPs estimator: workload-aware | logical tokens/s: 68719.7 | valid tokens/s: 68719.7 | physical tokens/s: 69588.5 | loss tokens/s: 53473.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.972888E-01 | lm-global-supervised-tokens: 1.305498E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.546743E-02 | lm-token-buffer-mib: 2.337522E+02 | global_load_balancing_loss: 1.001145E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11163.903 | params rms: 0.060449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:27:19.865799 | finish at 2026-10-17 23:25:15 +times across ranks (ms): + save-checkpoint: + rank 0: 24609.32 + rank 1: 24609.32 + rank 2: 24609.34 + rank 3: 24609.35 + rank 4: 24609.33 + rank 5: 24609.35 + rank 6: 24609.36 + rank 7: 24609.33 + rank 8: 24608.96 + rank 9: 24608.98 + rank 10: 24608.96 + rank 11: 24608.95 + rank 12: 24608.94 + rank 13: 24608.93 + rank 14: 24609.00 + rank 15: 24608.94 + rank 16: 24608.80 + rank 17: 24608.78 + rank 18: 24608.82 + rank 19: 24608.81 + rank 20: 24608.83 + rank 21: 24608.80 + rank 22: 24608.81 + rank 23: 24608.79 + rank 24: 24608.95 + rank 25: 24608.95 + rank 26: 24608.95 + rank 27: 24608.96 + rank 28: 24608.94 + rank 29: 24608.98 + rank 30: 24608.95 + rank 31: 24608.97 + rank 32: 24608.62 + rank 33: 24608.68 + rank 34: 24608.62 + rank 35: 24608.64 + rank 36: 24608.66 + rank 37: 24608.63 + rank 38: 24608.54 + rank 39: 24608.54 + rank 40: 24608.69 + rank 41: 24608.65 + rank 42: 24608.64 + rank 43: 24608.65 + rank 44: 24608.67 + rank 45: 24608.65 + rank 46: 24608.62 + rank 47: 24608.62 + rank 48: 24609.13 + rank 49: 24609.14 + rank 50: 24609.16 + rank 51: 24609.15 + rank 52: 24609.14 + rank 53: 24609.16 + rank 54: 24609.13 + rank 55: 24609.12 + rank 56: 24608.79 + rank 57: 24608.82 + rank 58: 24608.82 + rank 59: 24608.80 + rank 60: 24608.83 + rank 61: 24608.79 + rank 62: 24608.79 + rank 63: 24608.79 + [2026-10-15 21:02:07] iteration 1641/ 2384 | consumed samples: 52512 | elapsed time per iteration (ms): 226917.2 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.67% | FLOPs estimator: workload-aware | logical tokens/s: 73935.2 | valid tokens/s: 73935.2 | physical tokens/s: 74379.1 | loss tokens/s: 57076.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.592227E-01 | lm-global-supervised-tokens: 1.295151E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.114378E-01 | lm-token-buffer-mib: 2.335431E+02 | global_load_balancing_loss: 1.001195E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11163.868 | params rms: 0.060449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:49:59.452385 | finish at 2026-10-17 19:52:07 + [2026-10-15 21:05:55] iteration 1642/ 2384 | consumed samples: 52544 | elapsed time per iteration (ms): 227425.7 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.13% | FLOPs estimator: workload-aware | logical tokens/s: 73770.1 | valid tokens/s: 73770.1 | physical tokens/s: 74378.8 | loss tokens/s: 57590.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.069745E-01 | lm-global-supervised-tokens: 1.309764E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.717461E-02 | lm-token-buffer-mib: 2.335736E+02 | global_load_balancing_loss: 1.001042E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11163.835 | params rms: 0.060449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:52:29.902090 | finish at 2026-10-17 19:58:25 + [2026-10-15 21:09:39] iteration 1643/ 2384 | consumed samples: 52576 | elapsed time per iteration (ms): 223974.0 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 74906.8 | valid tokens/s: 74906.8 | physical tokens/s: 75589.1 | loss tokens/s: 58895.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.060059E-01 | lm-global-supervised-tokens: 1.319100E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.732527E-02 | lm-token-buffer-mib: 2.339486E+02 | global_load_balancing_loss: 1.000361E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11163.807 | params rms: 0.060448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:06:04.703773 | finish at 2026-10-17 19:15:43 + [2026-10-15 21:13:21] iteration 1644/ 2384 | consumed samples: 52608 | elapsed time per iteration (ms): 221885.6 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75611.9 | valid tokens/s: 75611.9 | physical tokens/s: 76210.2 | loss tokens/s: 53942.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.761885E-01 | lm-global-supervised-tokens: 1.196912E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.257338E-02 | lm-token-buffer-mib: 2.348543E+02 | global_load_balancing_loss: 1.000720E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11163.780 | params rms: 0.060448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:36:35.308604 | finish at 2026-10-17 18:49:56 + [2026-10-15 21:17:01] iteration 1645/ 2384 | consumed samples: 52640 | elapsed time per iteration (ms): 219975.6 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.82% | FLOPs estimator: workload-aware | logical tokens/s: 76268.5 | valid tokens/s: 76268.5 | physical tokens/s: 76775.1 | loss tokens/s: 61475.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.983437E-01 | lm-global-supervised-tokens: 1.352300E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.270705E-02 | lm-token-buffer-mib: 2.333116E+02 | global_load_balancing_loss: 1.000165E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11163.749 | params rms: 0.060448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:09:21.937393 | finish at 2026-10-17 18:26:23 + [2026-10-15 21:20:41] iteration 1646/ 2384 | consumed samples: 52672 | elapsed time per iteration (ms): 220493.3 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 76089.4 | valid tokens/s: 76089.4 | physical tokens/s: 76491.5 | loss tokens/s: 54369.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.678120E-01 | lm-global-supervised-tokens: 1.198807E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.621988E-02 | lm-token-buffer-mib: 2.335460E+02 | global_load_balancing_loss: 9.990630E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11163.713 | params rms: 0.060448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:12:04.077189 | finish at 2026-10-17 18:32:45 + [2026-10-15 21:24:23] iteration 1647/ 2384 | consumed samples: 52704 | elapsed time per iteration (ms): 222066.4 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75550.3 | valid tokens/s: 75550.3 | physical tokens/s: 76161.4 | loss tokens/s: 58365.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.364552E-01 | lm-global-supervised-tokens: 1.296108E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.529630E-02 | lm-token-buffer-mib: 2.345125E+02 | global_load_balancing_loss: 9.986477E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11163.680 | params rms: 0.060448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:27:42.924538 | finish at 2026-10-17 18:52:06 + [2026-10-15 21:28:02] iteration 1648/ 2384 | consumed samples: 52736 | elapsed time per iteration (ms): 218355.0 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.74% | FLOPs estimator: workload-aware | logical tokens/s: 76834.3 | valid tokens/s: 76834.3 | physical tokens/s: 77198.6 | loss tokens/s: 56826.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.109925E-01 | lm-global-supervised-tokens: 1.240837E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.271508E-02 | lm-token-buffer-mib: 2.324753E+02 | global_load_balancing_loss: 9.992774E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11163.654 | params rms: 0.060448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:38:29.314934 | finish at 2026-10-17 18:06:31 + [2026-10-15 21:31:44] iteration 1649/ 2384 | consumed samples: 52768 | elapsed time per iteration (ms): 222749.8 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.66% | FLOPs estimator: workload-aware | logical tokens/s: 75318.6 | valid tokens/s: 75318.6 | physical tokens/s: 76120.0 | loss tokens/s: 57814.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.744648E-01 | lm-global-supervised-tokens: 1.287815E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.765582E-02 | lm-token-buffer-mib: 2.351690E+02 | global_load_balancing_loss: 1.001276E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11163.623 | params rms: 0.060447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:28:41.072484 | finish at 2026-10-17 19:00:25 + [2026-10-15 21:35:45] iteration 1650/ 2384 | consumed samples: 52800 | elapsed time per iteration (ms): 241191.0 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 69559.8 | valid tokens/s: 69559.8 | physical tokens/s: 70038.0 | loss tokens/s: 55244.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.918230E-01 | lm-global-supervised-tokens: 1.332436E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.302126E-02 | lm-token-buffer-mib: 2.325098E+02 | global_load_balancing_loss: 1.000314E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11163.596 | params rms: 0.060447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:10:34.171939 | finish at 2026-10-17 22:46:20 + [2026-10-15 21:39:28] iteration 1651/ 2384 | consumed samples: 52832 | elapsed time per iteration (ms): 222026.7 | throughput per GPU (TFLOP/s/GPU): 27.1 | MFU 8.69% | FLOPs estimator: workload-aware | logical tokens/s: 75563.8 | valid tokens/s: 75563.8 | physical tokens/s: 76264.1 | loss tokens/s: 55473.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.150899E-01 | lm-global-supervised-tokens: 1.231665E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.377533E-02 | lm-token-buffer-mib: 2.340492E+02 | global_load_balancing_loss: 1.001128E+00 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11163.568 | params rms: 0.060447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:12:25.599775 | finish at 2026-10-17 18:51:53 + [2026-10-15 21:43:09] iteration 1652/ 2384 | consumed samples: 52864 | elapsed time per iteration (ms): 221963.7 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.03% | FLOPs estimator: workload-aware | logical tokens/s: 75585.2 | valid tokens/s: 75585.2 | physical tokens/s: 76237.9 | loss tokens/s: 57513.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.706677E-01 | lm-global-supervised-tokens: 1.276588E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.199882E-02 | lm-token-buffer-mib: 2.336302E+02 | global_load_balancing_loss: 1.001182E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11163.540 | params rms: 0.060447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:07:57.436469 | finish at 2026-10-17 18:51:07 + [2026-10-15 21:46:54] iteration 1653/ 2384 | consumed samples: 52896 | elapsed time per iteration (ms): 224248.3 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 74815.3 | valid tokens/s: 74815.3 | physical tokens/s: 75762.3 | loss tokens/s: 57156.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.831245E-01 | lm-global-supervised-tokens: 1.281718E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.430407E-02 | lm-token-buffer-mib: 2.364626E+02 | global_load_balancing_loss: 1.002122E+00 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11163.515 | params rms: 0.060447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:32:05.498467 | finish at 2026-10-17 19:18:59 + [2026-10-15 21:50:41] iteration 1654/ 2384 | consumed samples: 52928 | elapsed time per iteration (ms): 227621.3 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 73706.6 | valid tokens/s: 73706.6 | physical tokens/s: 74749.6 | loss tokens/s: 59128.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.716003E-01 | lm-global-supervised-tokens: 1.345882E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.333896E-02 | lm-token-buffer-mib: 2.345704E+02 | global_load_balancing_loss: 1.000683E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11163.492 | params rms: 0.060447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:09:23.561170 | finish at 2026-10-17 20:00:05 + [2026-10-15 21:54:28] iteration 1655/ 2384 | consumed samples: 52960 | elapsed time per iteration (ms): 226729.9 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 73996.1 | valid tokens/s: 73996.1 | physical tokens/s: 74961.0 | loss tokens/s: 56660.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.874749E-01 | lm-global-supervised-tokens: 1.284656E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.803181E-02 | lm-token-buffer-mib: 2.352471E+02 | global_load_balancing_loss: 9.998909E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11163.470 | params rms: 0.060447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:54:46.125693 | finish at 2026-10-17 19:49:14 + [2026-10-15 21:58:13] iteration 1656/ 2384 | consumed samples: 52992 | elapsed time per iteration (ms): 225227.3 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.02% | FLOPs estimator: workload-aware | logical tokens/s: 74490.1 | valid tokens/s: 74490.1 | physical tokens/s: 75136.6 | loss tokens/s: 54330.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.820808E-01 | lm-global-supervised-tokens: 1.223675E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.295979E-02 | lm-token-buffer-mib: 2.333616E+02 | global_load_balancing_loss: 9.996611E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11163.444 | params rms: 0.060446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:32:45.439808 | finish at 2026-10-17 19:30:59 + [2026-10-15 22:01:55] iteration 1657/ 2384 | consumed samples: 53024 | elapsed time per iteration (ms): 222047.3 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 75556.7 | valid tokens/s: 75556.7 | physical tokens/s: 75924.4 | loss tokens/s: 59516.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.589117E-01 | lm-global-supervised-tokens: 1.321537E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.472273E-02 | lm-token-buffer-mib: 2.322669E+02 | global_load_balancing_loss: 9.999770E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11163.417 | params rms: 0.060446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:50:28.406239 | finish at 2026-10-17 18:52:24 + [2026-10-15 22:05:38] iteration 1658/ 2384 | consumed samples: 53056 | elapsed time per iteration (ms): 222330.6 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 75460.5 | valid tokens/s: 75460.5 | physical tokens/s: 76210.3 | loss tokens/s: 53600.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.279137E-01 | lm-global-supervised-tokens: 1.191695E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.679681E-02 | lm-token-buffer-mib: 2.336868E+02 | global_load_balancing_loss: 9.990127E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11163.392 | params rms: 0.060446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:50:12.005751 | finish at 2026-10-17 18:55:50 + [2026-10-15 22:09:22] iteration 1659/ 2384 | consumed samples: 53088 | elapsed time per iteration (ms): 223916.9 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 74925.9 | valid tokens/s: 74925.9 | physical tokens/s: 76252.0 | loss tokens/s: 59716.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.993735E-01 | lm-global-supervised-tokens: 1.337153E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.199350E-02 | lm-token-buffer-mib: 2.359268E+02 | global_load_balancing_loss: 1.000244E+00 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11163.366 | params rms: 0.060446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:05:39.780444 | finish at 2026-10-17 19:15:01 + [2026-10-15 22:13:29] iteration 1660/ 2384 | consumed samples: 53120 | elapsed time per iteration (ms): 246912.5 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 67947.9 | valid tokens/s: 67947.9 | physical tokens/s: 68848.8 | loss tokens/s: 50736.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.225377E-01 | lm-global-supervised-tokens: 1.252744E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.323498E-02 | lm-token-buffer-mib: 2.370051E+02 | global_load_balancing_loss: 9.993770E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11163.339 | params rms: 0.060446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:39:24.621449 | finish at 2026-10-17 23:52:53 + [2026-10-15 22:17:08] iteration 1661/ 2384 | consumed samples: 53152 | elapsed time per iteration (ms): 219852.6 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 76311.0 | valid tokens/s: 76311.0 | physical tokens/s: 76990.0 | loss tokens/s: 58665.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.939252E-01 | lm-global-supervised-tokens: 1.289779E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.014152E-02 | lm-token-buffer-mib: 2.335233E+02 | global_load_balancing_loss: 1.000887E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11163.313 | params rms: 0.060446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:09:13.446591 | finish at 2026-10-17 18:26:22 + [2026-10-15 22:20:48] iteration 1662/ 2384 | consumed samples: 53184 | elapsed time per iteration (ms): 219822.0 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 76321.6 | valid tokens/s: 76321.6 | physical tokens/s: 77138.5 | loss tokens/s: 59570.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.258815E-01 | lm-global-supervised-tokens: 1.309496E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.261280E-02 | lm-token-buffer-mib: 2.336057E+02 | global_load_balancing_loss: 1.001829E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11163.287 | params rms: 0.060446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:05:11.473347 | finish at 2026-10-17 18:26:00 + [2026-10-15 22:24:29] iteration 1663/ 2384 | consumed samples: 53216 | elapsed time per iteration (ms): 221291.3 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75815.1 | valid tokens/s: 75815.1 | physical tokens/s: 76675.2 | loss tokens/s: 54057.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.545366E-01 | lm-global-supervised-tokens: 1.196240E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.420351E-02 | lm-token-buffer-mib: 2.334861E+02 | global_load_balancing_loss: 1.001632E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11163.261 | params rms: 0.060445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:19:10.996916 | finish at 2026-10-17 18:43:40 + [2026-10-15 22:28:11] iteration 1664/ 2384 | consumed samples: 53248 | elapsed time per iteration (ms): 221617.4 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 75703.4 | valid tokens/s: 75703.4 | physical tokens/s: 76241.5 | loss tokens/s: 57058.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.868587E-01 | lm-global-supervised-tokens: 1.264524E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.930377E-02 | lm-token-buffer-mib: 2.335970E+02 | global_load_balancing_loss: 1.000430E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11163.230 | params rms: 0.060445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:19:24.549408 | finish at 2026-10-17 18:47:36 + [2026-10-15 22:31:53] iteration 1665/ 2384 | consumed samples: 53280 | elapsed time per iteration (ms): 221954.3 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.26% | FLOPs estimator: workload-aware | logical tokens/s: 75588.5 | valid tokens/s: 75588.5 | physical tokens/s: 75926.8 | loss tokens/s: 52336.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.887035E-01 | lm-global-supervised-tokens: 1.161630E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.137957E-02 | lm-token-buffer-mib: 2.329753E+02 | global_load_balancing_loss: 1.001003E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11163.199 | params rms: 0.060445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:19:45.137533 | finish at 2026-10-17 18:51:38 + [2026-10-15 22:35:37] iteration 1666/ 2384 | consumed samples: 53312 | elapsed time per iteration (ms): 223769.1 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 74975.5 | valid tokens/s: 74975.5 | physical tokens/s: 75626.5 | loss tokens/s: 57337.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.055722E-01 | lm-global-supervised-tokens: 1.283039E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.239872E-02 | lm-token-buffer-mib: 2.338348E+02 | global_load_balancing_loss: 1.000229E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11163.169 | params rms: 0.060445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:37:46.207602 | finish at 2026-10-17 19:13:23 + [2026-10-15 22:39:19] iteration 1667/ 2384 | consumed samples: 53344 | elapsed time per iteration (ms): 222357.0 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.82% | FLOPs estimator: workload-aware | logical tokens/s: 75451.6 | valid tokens/s: 75451.6 | physical tokens/s: 75833.7 | loss tokens/s: 57433.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.630339E-01 | lm-global-supervised-tokens: 1.277072E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.875229E-02 | lm-token-buffer-mib: 2.330319E+02 | global_load_balancing_loss: 9.976596E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11163.139 | params rms: 0.060445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:17:09.959508 | finish at 2026-10-17 18:56:29 + [2026-10-15 22:43:05] iteration 1668/ 2384 | consumed samples: 53376 | elapsed time per iteration (ms): 225485.2 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 74404.8 | valid tokens/s: 74404.8 | physical tokens/s: 75179.8 | loss tokens/s: 55270.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.032770E-01 | lm-global-supervised-tokens: 1.246276E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.318662E-02 | lm-token-buffer-mib: 2.353250E+02 | global_load_balancing_loss: 9.983108E-01 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11163.108 | params rms: 0.060445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:50:47.435754 | finish at 2026-10-17 19:33:52 + [2026-10-15 22:46:50] iteration 1669/ 2384 | consumed samples: 53408 | elapsed time per iteration (ms): 225402.3 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 74432.2 | valid tokens/s: 74432.2 | physical tokens/s: 75187.9 | loss tokens/s: 52974.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.910423E-01 | lm-global-supervised-tokens: 1.194059E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.172472E-02 | lm-token-buffer-mib: 2.353316E+02 | global_load_balancing_loss: 1.000227E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11163.075 | params rms: 0.060444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:46:02.638448 | finish at 2026-10-17 19:32:53 + [2026-10-15 22:51:00] iteration 1670/ 2384 | consumed samples: 53440 | elapsed time per iteration (ms): 250336.4 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.11% | FLOPs estimator: workload-aware | logical tokens/s: 67018.7 | valid tokens/s: 67018.6 | physical tokens/s: 67981.7 | loss tokens/s: 50479.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.271255E-01 | lm-global-supervised-tokens: 1.263693E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.868245E-02 | lm-token-buffer-mib: 2.363080E+02 | global_load_balancing_loss: 1.001685E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11163.041 | params rms: 0.060444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:39:00.214647 | finish at 2026-10-18 00:30:01 + [2026-10-15 22:54:43] iteration 1671/ 2384 | consumed samples: 53472 | elapsed time per iteration (ms): 223069.2 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75210.6 | valid tokens/s: 75210.6 | physical tokens/s: 75756.5 | loss tokens/s: 60787.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.364605E-01 | lm-global-supervised-tokens: 1.355980E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.208711E-02 | lm-token-buffer-mib: 2.326280E+02 | global_load_balancing_loss: 1.000807E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11163.010 | params rms: 0.060444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:10:48.329598 | finish at 2026-10-17 19:05:32 + [2026-10-15 22:58:27] iteration 1672/ 2384 | consumed samples: 53504 | elapsed time per iteration (ms): 223366.7 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 75110.4 | valid tokens/s: 75110.4 | physical tokens/s: 75701.7 | loss tokens/s: 59775.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.398298E-01 | lm-global-supervised-tokens: 1.335188E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.224615E-02 | lm-token-buffer-mib: 2.340317E+02 | global_load_balancing_loss: 1.001367E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11162.978 | params rms: 0.060444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:10:37.107498 | finish at 2026-10-17 19:09:04 + [2026-10-15 23:02:08] iteration 1673/ 2384 | consumed samples: 53536 | elapsed time per iteration (ms): 220879.9 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.48% | FLOPs estimator: workload-aware | logical tokens/s: 75956.2 | valid tokens/s: 75956.2 | physical tokens/s: 76375.8 | loss tokens/s: 60330.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.057370E-01 | lm-global-supervised-tokens: 1.332568E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.581454E-02 | lm-token-buffer-mib: 2.330871E+02 | global_load_balancing_loss: 1.000990E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11162.944 | params rms: 0.060444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:37:25.617191 | finish at 2026-10-17 18:39:33 + [2026-10-15 23:05:49] iteration 1674/ 2384 | consumed samples: 53568 | elapsed time per iteration (ms): 221496.4 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 75744.8 | valid tokens/s: 75744.8 | physical tokens/s: 76590.3 | loss tokens/s: 61083.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.135680E-01 | lm-global-supervised-tokens: 1.352966E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.385765E-02 | lm-token-buffer-mib: 2.337858E+02 | global_load_balancing_loss: 1.000230E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11162.911 | params rms: 0.060444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:41:02.448485 | finish at 2026-10-17 18:46:52 + [2026-10-15 23:09:30] iteration 1675/ 2384 | consumed samples: 53600 | elapsed time per iteration (ms): 220732.4 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 76006.9 | valid tokens/s: 76006.9 | physical tokens/s: 76444.0 | loss tokens/s: 52370.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.869927E-01 | lm-global-supervised-tokens: 1.155978E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.199517E-02 | lm-token-buffer-mib: 2.340290E+02 | global_load_balancing_loss: 1.001508E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11162.880 | params rms: 0.060443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:28:19.287955 | finish at 2026-10-17 18:37:49 + [2026-10-15 23:13:11] iteration 1676/ 2384 | consumed samples: 53632 | elapsed time per iteration (ms): 220802.5 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75982.8 | valid tokens/s: 75982.8 | physical tokens/s: 76802.6 | loss tokens/s: 57469.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.630567E-01 | lm-global-supervised-tokens: 1.268947E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.893396E-02 | lm-token-buffer-mib: 2.355288E+02 | global_load_balancing_loss: 9.997043E-01 | loss scale: 1.0 | grad norm: 0.239 | params norm: 11162.850 | params rms: 0.060443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:25:28.142830 | finish at 2026-10-17 18:38:39 + [2026-10-15 23:16:50] iteration 1677/ 2384 | consumed samples: 53664 | elapsed time per iteration (ms): 219290.5 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 76506.7 | valid tokens/s: 76506.7 | physical tokens/s: 77265.6 | loss tokens/s: 58495.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.160413E-01 | lm-global-supervised-tokens: 1.282744E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.752626E-02 | lm-token-buffer-mib: 2.341983E+02 | global_load_balancing_loss: 9.994338E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11162.817 | params rms: 0.060443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:03:58.370637 | finish at 2026-10-17 18:20:48 + [2026-10-15 23:20:33] iteration 1678/ 2384 | consumed samples: 53696 | elapsed time per iteration (ms): 223119.0 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75194.0 | valid tokens/s: 75194.0 | physical tokens/s: 75628.9 | loss tokens/s: 58019.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.324198E-01 | lm-global-supervised-tokens: 1.294528E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.179825E-02 | lm-token-buffer-mib: 2.335384E+02 | global_load_balancing_loss: 9.999282E-01 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11162.785 | params rms: 0.060443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:45:21.996465 | finish at 2026-10-17 19:05:55 + [2026-10-15 23:24:13] iteration 1679/ 2384 | consumed samples: 53728 | elapsed time per iteration (ms): 219876.3 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.26% | FLOPs estimator: workload-aware | logical tokens/s: 76302.8 | valid tokens/s: 76302.8 | physical tokens/s: 76617.0 | loss tokens/s: 56818.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.519364E-01 | lm-global-supervised-tokens: 1.249295E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.839538E-02 | lm-token-buffer-mib: 2.335220E+02 | global_load_balancing_loss: 9.996280E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11162.755 | params rms: 0.060443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:03:32.793249 | finish at 2026-10-17 18:27:46 + [2026-10-15 23:28:25] iteration 1680/ 2384 | consumed samples: 53760 | elapsed time per iteration (ms): 252091.8 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 66551.8 | valid tokens/s: 66551.8 | physical tokens/s: 68298.7 | loss tokens/s: 48320.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.562244E-01 | lm-global-supervised-tokens: 1.218109E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.322675E-02 | lm-token-buffer-mib: 2.409679E+02 | global_load_balancing_loss: 1.000170E+00 | loss scale: 1.0 | grad norm: 0.240 | params norm: 11162.723 | params rms: 0.060443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:17:52.661926 | finish at 2026-10-18 00:46:18 +times across ranks (ms): + save-checkpoint: + rank 0: 23974.50 + rank 1: 23974.51 + rank 2: 23974.53 + rank 3: 23974.52 + rank 4: 23974.48 + rank 5: 23974.49 + rank 6: 23974.53 + rank 7: 23974.48 + rank 8: 23973.97 + rank 9: 23974.00 + rank 10: 23974.01 + rank 11: 23973.99 + rank 12: 23974.01 + rank 13: 23973.97 + rank 14: 23973.97 + rank 15: 23974.06 + rank 16: 23974.50 + rank 17: 23974.49 + rank 18: 23974.51 + rank 19: 23974.52 + rank 20: 23974.45 + rank 21: 23974.44 + rank 22: 23974.49 + rank 23: 23974.45 + rank 24: 23974.48 + rank 25: 23974.67 + rank 26: 23974.77 + rank 27: 23974.49 + rank 28: 23974.49 + rank 29: 23974.47 + rank 30: 23974.49 + rank 31: 23974.50 + rank 32: 23974.21 + rank 33: 23974.21 + rank 34: 23974.18 + rank 35: 23974.18 + rank 36: 23974.17 + rank 37: 23974.21 + rank 38: 23974.17 + rank 39: 23974.16 + rank 40: 23974.55 + rank 41: 23974.59 + rank 42: 23974.53 + rank 43: 23974.57 + rank 44: 23974.57 + rank 45: 23974.57 + rank 46: 23974.59 + rank 47: 23974.57 + rank 48: 23974.36 + rank 49: 23974.41 + rank 50: 23974.41 + rank 51: 23974.37 + rank 52: 23974.37 + rank 53: 23974.39 + rank 54: 23974.38 + rank 55: 23974.34 + rank 56: 23974.57 + rank 57: 23974.56 + rank 58: 23974.56 + rank 59: 23974.55 + rank 60: 23974.59 + rank 61: 23974.54 + rank 62: 23974.53 + rank 63: 23974.54 + [2026-10-15 23:32:31] iteration 1681/ 2384 | consumed samples: 53792 | elapsed time per iteration (ms): 221616.7 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 75703.5 | valid tokens/s: 75703.5 | physical tokens/s: 76069.9 | loss tokens/s: 55373.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.156629E-01 | lm-global-supervised-tokens: 1.227164E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.475286E-02 | lm-token-buffer-mib: 2.335201E+02 | global_load_balancing_loss: 1.000979E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11162.689 | params rms: 0.060442 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:16:36.560502 | finish at 2026-10-17 18:49:07 + [2026-10-15 23:36:16] iteration 1682/ 2384 | consumed samples: 53824 | elapsed time per iteration (ms): 225075.4 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.26% | FLOPs estimator: workload-aware | logical tokens/s: 74540.3 | valid tokens/s: 74540.3 | physical tokens/s: 74810.3 | loss tokens/s: 57119.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.504975E-01 | lm-global-supervised-tokens: 1.285619E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.568681E-02 | lm-token-buffer-mib: 2.334301E+02 | global_load_balancing_loss: 1.001300E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11162.656 | params rms: 0.060442 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:53:22.907448 | finish at 2026-10-17 19:29:39 + [2026-10-15 23:39:57] iteration 1683/ 2384 | consumed samples: 53856 | elapsed time per iteration (ms): 221533.2 | throughput per GPU (TFLOP/s/GPU): 27.0 | MFU 8.65% | FLOPs estimator: workload-aware | logical tokens/s: 75732.3 | valid tokens/s: 75732.2 | physical tokens/s: 76518.4 | loss tokens/s: 54608.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.483983E-01 | lm-global-supervised-tokens: 1.209759E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.585705E-02 | lm-token-buffer-mib: 2.334001E+02 | global_load_balancing_loss: 1.001169E+00 | loss scale: 1.0 | grad norm: 0.293 | params norm: 11162.625 | params rms: 0.060442 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:08:14.773886 | finish at 2026-10-17 18:48:12 + [2026-10-15 23:43:39] iteration 1684/ 2384 | consumed samples: 53888 | elapsed time per iteration (ms): 222033.7 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 75561.5 | valid tokens/s: 75561.5 | physical tokens/s: 76562.9 | loss tokens/s: 60784.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.091931E-01 | lm-global-supervised-tokens: 1.349621E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.212465E-02 | lm-token-buffer-mib: 2.363809E+02 | global_load_balancing_loss: 1.000524E+00 | loss scale: 1.0 | grad norm: 0.170 | params norm: 11162.588 | params rms: 0.060442 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:10:23.599505 | finish at 2026-10-17 18:54:03 + [2026-10-15 23:47:23] iteration 1685/ 2384 | consumed samples: 53920 | elapsed time per iteration (ms): 223959.4 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 74911.9 | valid tokens/s: 74911.9 | physical tokens/s: 75855.8 | loss tokens/s: 58425.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.376019E-01 | lm-global-supervised-tokens: 1.308492E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.893193E-02 | lm-token-buffer-mib: 2.359466E+02 | global_load_balancing_loss: 9.995196E-01 | loss scale: 1.0 | grad norm: 0.380 | params norm: 11162.551 | params rms: 0.060442 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:29:07.612224 | finish at 2026-10-17 19:16:31 + [2026-10-15 23:51:05] iteration 1686/ 2384 | consumed samples: 53952 | elapsed time per iteration (ms): 221713.0 | throughput per GPU (TFLOP/s/GPU): 25.5 | MFU 8.16% | FLOPs estimator: workload-aware | logical tokens/s: 75670.7 | valid tokens/s: 75670.7 | physical tokens/s: 76125.8 | loss tokens/s: 60410.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.651581E-01 | lm-global-supervised-tokens: 1.339384E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.213670E-02 | lm-token-buffer-mib: 2.338972E+02 | global_load_balancing_loss: 9.981304E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11162.515 | params rms: 0.060441 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:59:15.699503 | finish at 2026-10-17 18:50:21 + [2026-10-15 23:54:46] iteration 1687/ 2384 | consumed samples: 53984 | elapsed time per iteration (ms): 220974.5 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 75923.5 | valid tokens/s: 75923.5 | physical tokens/s: 76517.2 | loss tokens/s: 59822.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.837573E-01 | lm-global-supervised-tokens: 1.321914E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.130222E-02 | lm-token-buffer-mib: 2.340303E+02 | global_load_balancing_loss: 9.997321E-01 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11162.481 | params rms: 0.060441 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:46:59.212666 | finish at 2026-10-17 18:41:45 + [2026-10-15 23:58:28] iteration 1688/ 2384 | consumed samples: 54016 | elapsed time per iteration (ms): 222321.4 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 75463.5 | valid tokens/s: 75463.5 | physical tokens/s: 76164.5 | loss tokens/s: 58255.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.574946E-01 | lm-global-supervised-tokens: 1.295153E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.908221E-02 | lm-token-buffer-mib: 2.347753E+02 | global_load_balancing_loss: 1.002226E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11162.447 | params rms: 0.060441 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:58:55.703642 | finish at 2026-10-17 18:57:24 + [2026-10-16 00:02:13] iteration 1689/ 2384 | consumed samples: 54048 | elapsed time per iteration (ms): 224067.1 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 74875.7 | valid tokens/s: 74875.7 | physical tokens/s: 75905.0 | loss tokens/s: 59102.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.296987E-01 | lm-global-supervised-tokens: 1.324295E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.271933E-02 | lm-token-buffer-mib: 2.354299E+02 | global_load_balancing_loss: 1.002574E+00 | loss scale: 1.0 | grad norm: 0.465 | params norm: 11162.411 | params rms: 0.060441 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:15:26.642321 | finish at 2026-10-17 19:17:39 + [2026-10-16 00:06:18] iteration 1690/ 2384 | consumed samples: 54080 | elapsed time per iteration (ms): 245858.4 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 68239.1 | valid tokens/s: 68239.1 | physical tokens/s: 68668.9 | loss tokens/s: 54737.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.751009E-01 | lm-global-supervised-tokens: 1.345773E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.205898E-02 | lm-token-buffer-mib: 2.330885E+02 | global_load_balancing_loss: 1.001358E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11162.375 | params rms: 0.060441 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:23:45.698071 | finish at 2026-10-17 23:30:04 + [2026-10-16 00:10:00] iteration 1691/ 2384 | consumed samples: 54112 | elapsed time per iteration (ms): 222005.6 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 75571.1 | valid tokens/s: 75571.1 | physical tokens/s: 76471.1 | loss tokens/s: 59404.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.717352E-01 | lm-global-supervised-tokens: 1.318807E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.913213E-02 | lm-token-buffer-mib: 2.346677E+02 | global_load_balancing_loss: 1.001776E+00 | loss scale: 1.0 | grad norm: 0.359 | params norm: 11162.338 | params rms: 0.060440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:44:09.904080 | finish at 2026-10-17 18:54:10 + [2026-10-16 00:13:47] iteration 1692/ 2384 | consumed samples: 54144 | elapsed time per iteration (ms): 226369.5 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 74114.1 | valid tokens/s: 74114.1 | physical tokens/s: 74653.7 | loss tokens/s: 50482.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.352727E-01 | lm-global-supervised-tokens: 1.142771E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.171239E-02 | lm-token-buffer-mib: 2.344780E+02 | global_load_balancing_loss: 9.996902E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11162.300 | params rms: 0.060440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:30:47.713579 | finish at 2026-10-17 19:44:34 + [2026-10-16 00:17:30] iteration 1693/ 2384 | consumed samples: 54176 | elapsed time per iteration (ms): 223693.4 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 75000.8 | valid tokens/s: 75000.8 | physical tokens/s: 76272.1 | loss tokens/s: 56664.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.856526E-01 | lm-global-supervised-tokens: 1.267555E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.252029E-02 | lm-token-buffer-mib: 2.367403E+02 | global_load_balancing_loss: 9.996975E-01 | loss scale: 1.0 | grad norm: 0.228 | params norm: 11162.261 | params rms: 0.060440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:56:12.115447 | finish at 2026-10-17 19:13:43 + [2026-10-16 00:21:17] iteration 1694/ 2384 | consumed samples: 54208 | elapsed time per iteration (ms): 226523.1 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.98% | FLOPs estimator: workload-aware | logical tokens/s: 74064.0 | valid tokens/s: 74064.0 | physical tokens/s: 74741.3 | loss tokens/s: 61091.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.452075E-01 | lm-global-supervised-tokens: 1.383862E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.459721E-02 | lm-token-buffer-mib: 2.353600E+02 | global_load_balancing_loss: 9.987934E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11162.221 | params rms: 0.060440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:25:00.949788 | finish at 2026-10-17 19:46:18 + [2026-10-16 00:25:03] iteration 1695/ 2384 | consumed samples: 54240 | elapsed time per iteration (ms): 225761.5 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 74313.8 | valid tokens/s: 74313.8 | physical tokens/s: 75159.2 | loss tokens/s: 58313.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.207661E-01 | lm-global-supervised-tokens: 1.316484E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.201838E-02 | lm-token-buffer-mib: 2.347117E+02 | global_load_balancing_loss: 9.984533E-01 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11162.180 | params rms: 0.060440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:12:29.667341 | finish at 2026-10-17 19:37:32 + [2026-10-16 00:28:47] iteration 1696/ 2384 | consumed samples: 54272 | elapsed time per iteration (ms): 224488.3 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.21% | FLOPs estimator: workload-aware | logical tokens/s: 74735.3 | valid tokens/s: 74735.3 | physical tokens/s: 75830.4 | loss tokens/s: 56518.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.325896E-01 | lm-global-supervised-tokens: 1.268765E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.115141E-02 | lm-token-buffer-mib: 2.343340E+02 | global_load_balancing_loss: 1.000334E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11162.139 | params rms: 0.060439 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:54:07.921917 | finish at 2026-10-17 19:22:55 + [2026-10-16 00:32:29] iteration 1697/ 2384 | consumed samples: 54304 | elapsed time per iteration (ms): 221706.7 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 75672.9 | valid tokens/s: 75672.9 | physical tokens/s: 76717.7 | loss tokens/s: 57392.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.016748E-01 | lm-global-supervised-tokens: 1.272420E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.930404E-02 | lm-token-buffer-mib: 2.341711E+02 | global_load_balancing_loss: 1.001051E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11162.094 | params rms: 0.060439 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:18:32.469709 | finish at 2026-10-17 18:51:01 + [2026-10-16 00:36:07] iteration 1698/ 2384 | consumed samples: 54336 | elapsed time per iteration (ms): 218469.7 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 76794.2 | valid tokens/s: 76794.2 | physical tokens/s: 77691.2 | loss tokens/s: 57513.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.547371E-01 | lm-global-supervised-tokens: 1.256493E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.327478E-02 | lm-token-buffer-mib: 2.335788E+02 | global_load_balancing_loss: 1.001351E+00 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11162.051 | params rms: 0.060439 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:37:50.220646 | finish at 2026-10-17 18:13:58 + [2026-10-16 00:39:47] iteration 1699/ 2384 | consumed samples: 54368 | elapsed time per iteration (ms): 219811.3 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 76325.2 | valid tokens/s: 76325.2 | physical tokens/s: 76776.3 | loss tokens/s: 59111.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.477508E-01 | lm-global-supervised-tokens: 1.299340E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.028627E-02 | lm-token-buffer-mib: 2.337010E+02 | global_load_balancing_loss: 1.002331E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11162.014 | params rms: 0.060439 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:49:30.732688 | finish at 2026-10-17 18:29:18 + [2026-10-16 00:43:52] iteration 1700/ 2384 | consumed samples: 54400 | elapsed time per iteration (ms): 244968.8 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.24% | FLOPs estimator: workload-aware | logical tokens/s: 68487.0 | valid tokens/s: 68487.0 | physical tokens/s: 69123.1 | loss tokens/s: 51790.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.239283E-01 | lm-global-supervised-tokens: 1.268713E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.836129E-02 | lm-token-buffer-mib: 2.352455E+02 | global_load_balancing_loss: 1.001150E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11161.976 | params rms: 0.060438 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:32:38.633154 | finish at 2026-10-17 23:16:31 + [2026-10-16 00:47:35] iteration 1701/ 2384 | consumed samples: 54432 | elapsed time per iteration (ms): 222788.4 | throughput per GPU (TFLOP/s/GPU): 26.3 | MFU 8.43% | FLOPs estimator: workload-aware | logical tokens/s: 75305.3 | valid tokens/s: 75305.3 | physical tokens/s: 75683.9 | loss tokens/s: 60583.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.008944E-01 | lm-global-supervised-tokens: 1.349721E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.201497E-02 | lm-token-buffer-mib: 2.328389E+02 | global_load_balancing_loss: 1.000096E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11161.936 | params rms: 0.060438 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:16:04.502396 | finish at 2026-10-17 19:03:39 + [2026-10-16 00:51:19] iteration 1702/ 2384 | consumed samples: 54464 | elapsed time per iteration (ms): 224401.4 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.76% | FLOPs estimator: workload-aware | logical tokens/s: 74764.2 | valid tokens/s: 74764.2 | physical tokens/s: 75804.7 | loss tokens/s: 61289.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.037896E-01 | lm-global-supervised-tokens: 1.375333E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.355272E-02 | lm-token-buffer-mib: 2.373205E+02 | global_load_balancing_loss: 9.998179E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11161.894 | params rms: 0.060438 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:30:41.746406 | finish at 2026-10-17 19:22:01 + [2026-10-16 00:55:06] iteration 1703/ 2384 | consumed samples: 54496 | elapsed time per iteration (ms): 226380.6 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 74110.5 | valid tokens/s: 74110.5 | physical tokens/s: 75202.8 | loss tokens/s: 55939.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.292951E-01 | lm-global-supervised-tokens: 1.266350E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.553890E-02 | lm-token-buffer-mib: 2.365249E+02 | global_load_balancing_loss: 9.999705E-01 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11161.848 | params rms: 0.060438 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:49:25.156760 | finish at 2026-10-17 19:44:31 + [2026-10-16 00:58:50] iteration 1704/ 2384 | consumed samples: 54528 | elapsed time per iteration (ms): 224413.3 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 74760.3 | valid tokens/s: 74760.3 | physical tokens/s: 76020.9 | loss tokens/s: 56273.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.392819E-01 | lm-global-supervised-tokens: 1.262843E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.487647E-02 | lm-token-buffer-mib: 2.374680E+02 | global_load_balancing_loss: 9.998570E-01 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11161.804 | params rms: 0.060438 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:23:21.076937 | finish at 2026-10-17 19:22:11 + [2026-10-16 01:02:31] iteration 1705/ 2384 | consumed samples: 54560 | elapsed time per iteration (ms): 220916.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75943.7 | valid tokens/s: 75943.7 | physical tokens/s: 76793.5 | loss tokens/s: 58949.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.219961E-01 | lm-global-supervised-tokens: 1.302299E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.161149E-02 | lm-token-buffer-mib: 2.338957E+02 | global_load_balancing_loss: 9.988170E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11161.766 | params rms: 0.060437 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:40:02.021071 | finish at 2026-10-17 18:42:33 + [2026-10-16 01:06:17] iteration 1706/ 2384 | consumed samples: 54592 | elapsed time per iteration (ms): 225669.6 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.49% | FLOPs estimator: workload-aware | logical tokens/s: 74208.3 | valid tokens/s: 74208.3 | physical tokens/s: 74931.2 | loss tokens/s: 52728.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.725018E-01 | lm-global-supervised-tokens: 1.189923E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.894470E-02 | lm-token-buffer-mib: 2.342493E+02 | global_load_balancing_loss: 1.000614E+00 | loss scale: 1.0 | grad norm: 0.269 | params norm: 11161.731 | params rms: 0.060437 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:30:04.021621 | finish at 2026-10-17 19:36:21 + [2026-10-16 01:10:00] iteration 1707/ 2384 | consumed samples: 54624 | elapsed time per iteration (ms): 222888.6 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.38% | FLOPs estimator: workload-aware | logical tokens/s: 75271.7 | valid tokens/s: 75271.7 | physical tokens/s: 76480.3 | loss tokens/s: 61261.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.032213E-01 | lm-global-supervised-tokens: 1.365449E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.128936E-02 | lm-token-buffer-mib: 2.353397E+02 | global_load_balancing_loss: 9.994271E-01 | loss scale: 1.0 | grad norm: 0.210 | params norm: 11161.691 | params rms: 0.060437 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:54:55.554997 | finish at 2026-10-17 19:04:55 + [2026-10-16 01:13:40] iteration 1708/ 2384 | consumed samples: 54656 | elapsed time per iteration (ms): 220556.8 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.45% | FLOPs estimator: workload-aware | logical tokens/s: 76067.5 | valid tokens/s: 76067.5 | physical tokens/s: 76781.1 | loss tokens/s: 57312.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.415860E-01 | lm-global-supervised-tokens: 1.264057E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.250228E-02 | lm-token-buffer-mib: 2.329908E+02 | global_load_balancing_loss: 1.001000E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11161.651 | params rms: 0.060437 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:24:56.398981 | finish at 2026-10-17 18:38:37 + [2026-10-16 01:17:21] iteration 1709/ 2384 | consumed samples: 54688 | elapsed time per iteration (ms): 220606.2 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 76050.4 | valid tokens/s: 76050.3 | physical tokens/s: 76742.2 | loss tokens/s: 58021.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.247398E-01 | lm-global-supervised-tokens: 1.279998E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.106221E-02 | lm-token-buffer-mib: 2.330046E+02 | global_load_balancing_loss: 1.001390E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11161.612 | params rms: 0.060436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:21:49.158915 | finish at 2026-10-17 18:39:10 + [2026-10-16 01:21:27] iteration 1710/ 2384 | consumed samples: 54720 | elapsed time per iteration (ms): 246660.7 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.21% | FLOPs estimator: workload-aware | logical tokens/s: 68017.3 | valid tokens/s: 68017.3 | physical tokens/s: 68581.7 | loss tokens/s: 49233.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.723333E-01 | lm-global-supervised-tokens: 1.214388E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.109888E-02 | lm-token-buffer-mib: 2.341688E+02 | global_load_balancing_loss: 1.001505E+00 | loss scale: 1.0 | grad norm: 0.168 | params norm: 11161.575 | params rms: 0.060436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:10:49.312338 | finish at 2026-10-17 23:32:17 + [2026-10-16 01:25:08] iteration 1711/ 2384 | consumed samples: 54752 | elapsed time per iteration (ms): 220153.3 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 76206.9 | valid tokens/s: 76206.9 | physical tokens/s: 76974.7 | loss tokens/s: 53691.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.196325E-01 | lm-global-supervised-tokens: 1.182027E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.464868E-02 | lm-token-buffer-mib: 2.333757E+02 | global_load_balancing_loss: 1.002456E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11161.540 | params rms: 0.060436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:09:23.147666 | finish at 2026-10-17 18:34:31 + [2026-10-16 01:28:49] iteration 1712/ 2384 | consumed samples: 54784 | elapsed time per iteration (ms): 221175.7 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 75854.5 | valid tokens/s: 75854.5 | physical tokens/s: 76293.6 | loss tokens/s: 55301.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.707271E-01 | lm-global-supervised-tokens: 1.223138E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.189018E-02 | lm-token-buffer-mib: 2.328605E+02 | global_load_balancing_loss: 1.000705E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11161.505 | params rms: 0.060436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:17:10.095680 | finish at 2026-10-17 18:45:59 + [2026-10-16 01:32:28] iteration 1713/ 2384 | consumed samples: 54816 | elapsed time per iteration (ms): 219609.5 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 76395.7 | valid tokens/s: 76395.6 | physical tokens/s: 77298.8 | loss tokens/s: 60912.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.505609E-01 | lm-global-supervised-tokens: 1.337700E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.266880E-02 | lm-token-buffer-mib: 2.341211E+02 | global_load_balancing_loss: 1.000026E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11161.468 | params rms: 0.060436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:55:57.962935 | finish at 2026-10-17 18:28:26 + [2026-10-16 01:36:09] iteration 1714/ 2384 | consumed samples: 54848 | elapsed time per iteration (ms): 220886.4 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75953.9 | valid tokens/s: 75953.9 | physical tokens/s: 76352.4 | loss tokens/s: 56443.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.674317E-01 | lm-global-supervised-tokens: 1.246770E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.628762E-02 | lm-token-buffer-mib: 2.336378E+02 | global_load_balancing_loss: 9.979720E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11161.433 | params rms: 0.060436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:06:33.914187 | finish at 2026-10-17 18:42:43 + [2026-10-16 01:39:52] iteration 1715/ 2384 | consumed samples: 54880 | elapsed time per iteration (ms): 222797.6 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 75302.3 | valid tokens/s: 75302.3 | physical tokens/s: 75939.3 | loss tokens/s: 58405.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.204337E-01 | lm-global-supervised-tokens: 1.301252E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.379463E-02 | lm-token-buffer-mib: 2.326416E+02 | global_load_balancing_loss: 9.982745E-01 | loss scale: 1.0 | grad norm: 0.255 | params norm: 11161.398 | params rms: 0.060435 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:24:11.624088 | finish at 2026-10-17 19:04:04 + [2026-10-16 01:43:37] iteration 1716/ 2384 | consumed samples: 54912 | elapsed time per iteration (ms): 224629.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 74688.5 | valid tokens/s: 74688.5 | physical tokens/s: 75880.9 | loss tokens/s: 59950.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.063959E-01 | lm-global-supervised-tokens: 1.346669E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.263969E-02 | lm-token-buffer-mib: 2.369012E+02 | global_load_balancing_loss: 9.996238E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11161.365 | params rms: 0.060435 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:40:52.218630 | finish at 2026-10-17 19:24:29 + [2026-10-16 01:47:21] iteration 1717/ 2384 | consumed samples: 54944 | elapsed time per iteration (ms): 224691.0 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 74667.9 | valid tokens/s: 74667.9 | physical tokens/s: 75249.0 | loss tokens/s: 55977.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.859986E-01 | lm-global-supervised-tokens: 1.257766E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.197402E-02 | lm-token-buffer-mib: 2.334531E+02 | global_load_balancing_loss: 1.000941E+00 | loss scale: 1.0 | grad norm: 0.224 | params norm: 11161.332 | params rms: 0.060435 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:37:48.873295 | finish at 2026-10-17 19:25:10 + [2026-10-16 01:51:02] iteration 1718/ 2384 | consumed samples: 54976 | elapsed time per iteration (ms): 221069.9 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 75891.0 | valid tokens/s: 75891.0 | physical tokens/s: 76750.0 | loss tokens/s: 60636.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.543672E-01 | lm-global-supervised-tokens: 1.340482E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.169366E-02 | lm-token-buffer-mib: 2.353416E+02 | global_load_balancing_loss: 1.000994E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11161.297 | params rms: 0.060435 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:53:52.534369 | finish at 2026-10-17 18:44:55 + [2026-10-16 01:54:45] iteration 1719/ 2384 | consumed samples: 55008 | elapsed time per iteration (ms): 222418.8 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75430.6 | valid tokens/s: 75430.6 | physical tokens/s: 76356.7 | loss tokens/s: 54782.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.606375E-01 | lm-global-supervised-tokens: 1.218474E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.689241E-02 | lm-token-buffer-mib: 2.326953E+02 | global_load_balancing_loss: 1.002543E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11161.264 | params rms: 0.060435 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:05:08.506516 | finish at 2026-10-17 18:59:53 + [2026-10-16 01:58:54] iteration 1720/ 2384 | consumed samples: 55040 | elapsed time per iteration (ms): 248989.7 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.95% | FLOPs estimator: workload-aware | logical tokens/s: 67381.2 | valid tokens/s: 67381.2 | physical tokens/s: 68302.1 | loss tokens/s: 48149.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.126847E-01 | lm-global-supervised-tokens: 1.198879E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.688941E-02 | lm-token-buffer-mib: 2.352474E+02 | global_load_balancing_loss: 1.000570E+00 | loss scale: 1.0 | grad norm: 0.251 | params norm: 11161.231 | params rms: 0.060434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:55:29.147713 | finish at 2026-10-17 23:54:23 +times across ranks (ms): + save-checkpoint: + rank 0: 28447.34 + rank 1: 28447.42 + rank 2: 28447.35 + rank 3: 28447.35 + rank 4: 28447.33 + rank 5: 28447.36 + rank 6: 28447.35 + rank 7: 28447.27 + rank 8: 28449.71 + rank 9: 28449.71 + rank 10: 28449.75 + rank 11: 28449.69 + rank 12: 28449.73 + rank 13: 28449.68 + rank 14: 28449.77 + rank 15: 28449.69 + rank 16: 28447.40 + rank 17: 28447.39 + rank 18: 28447.50 + rank 19: 28447.38 + rank 20: 28447.35 + rank 21: 28447.37 + rank 22: 28447.41 + rank 23: 28447.20 + rank 24: 28448.30 + rank 25: 28448.31 + rank 26: 28448.34 + rank 27: 28448.97 + rank 28: 28448.26 + rank 29: 28448.33 + rank 30: 28448.28 + rank 31: 28448.28 + rank 32: 28449.37 + rank 33: 28449.44 + rank 34: 28449.42 + rank 35: 28449.46 + rank 36: 28449.40 + rank 37: 28449.45 + rank 38: 28449.39 + rank 39: 28449.40 + rank 40: 28447.49 + rank 41: 28447.47 + rank 42: 28447.47 + rank 43: 28447.48 + rank 44: 28447.46 + rank 45: 28447.46 + rank 46: 28447.50 + rank 47: 28447.46 + rank 48: 28446.19 + rank 49: 28446.16 + rank 50: 28446.20 + rank 51: 28446.06 + rank 52: 28446.07 + rank 53: 28446.18 + rank 54: 28446.14 + rank 55: 28446.11 + rank 56: 28447.90 + rank 57: 28448.01 + rank 58: 28447.98 + rank 59: 28447.94 + rank 60: 28447.94 + rank 61: 28447.88 + rank 62: 28447.99 + rank 63: 28447.89 + [2026-10-16 02:03:23] iteration 1721/ 2384 | consumed samples: 55072 | elapsed time per iteration (ms): 238697.4 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 70286.5 | valid tokens/s: 70286.4 | physical tokens/s: 70853.9 | loss tokens/s: 54868.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.530630E-01 | lm-global-supervised-tokens: 1.309699E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.424604E+00 | lm-token-buffer-mib: 2.339404E+02 | global_load_balancing_loss: 1.000747E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11161.198 | params rms: 0.060434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:57:36.383533 | finish at 2026-10-17 22:00:59 + [2026-10-16 02:07:10] iteration 1722/ 2384 | consumed samples: 55104 | elapsed time per iteration (ms): 227178.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 73850.3 | valid tokens/s: 73850.3 | physical tokens/s: 74676.6 | loss tokens/s: 56314.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.983872E-01 | lm-global-supervised-tokens: 1.279333E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.863575E-02 | lm-token-buffer-mib: 2.356373E+02 | global_load_balancing_loss: 1.000954E+00 | loss scale: 1.0 | grad norm: 0.363 | params norm: 11161.167 | params rms: 0.060434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:46:32.169326 | finish at 2026-10-17 19:53:42 + [2026-10-16 02:10:49] iteration 1723/ 2384 | consumed samples: 55136 | elapsed time per iteration (ms): 218608.2 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.20% | FLOPs estimator: workload-aware | logical tokens/s: 76745.5 | valid tokens/s: 76745.5 | physical tokens/s: 77794.9 | loss tokens/s: 59003.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.864420E-01 | lm-global-supervised-tokens: 1.289866E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.024573E-02 | lm-token-buffer-mib: 2.345781E+02 | global_load_balancing_loss: 9.997028E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11161.140 | params rms: 0.060434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:08:19.995979 | finish at 2026-10-17 18:19:09 + [2026-10-16 02:14:30] iteration 1724/ 2384 | consumed samples: 55168 | elapsed time per iteration (ms): 221934.8 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.44% | FLOPs estimator: workload-aware | logical tokens/s: 75595.1 | valid tokens/s: 75595.1 | physical tokens/s: 76240.9 | loss tokens/s: 53366.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.972216E-01 | lm-global-supervised-tokens: 1.184386E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.639260E-02 | lm-token-buffer-mib: 2.354642E+02 | global_load_balancing_loss: 9.989581E-01 | loss scale: 1.0 | grad norm: 0.212 | params norm: 11161.110 | params rms: 0.060434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:41:16.938043 | finish at 2026-10-17 18:55:47 + [2026-10-16 02:18:13] iteration 1725/ 2384 | consumed samples: 55200 | elapsed time per iteration (ms): 222341.9 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 75456.6 | valid tokens/s: 75456.6 | physical tokens/s: 76067.1 | loss tokens/s: 56496.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.578880E-01 | lm-global-supervised-tokens: 1.256155E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.484191E-02 | lm-token-buffer-mib: 2.334244E+02 | global_load_balancing_loss: 9.996052E-01 | loss scale: 1.0 | grad norm: 0.231 | params norm: 11161.077 | params rms: 0.060434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:42:03.315629 | finish at 2026-10-17 19:00:16 + [2026-10-16 02:21:57] iteration 1726/ 2384 | consumed samples: 55232 | elapsed time per iteration (ms): 223963.6 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 74910.4 | valid tokens/s: 74910.4 | physical tokens/s: 76057.8 | loss tokens/s: 58337.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.470901E-01 | lm-global-supervised-tokens: 1.306558E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.192474E-02 | lm-token-buffer-mib: 2.347613E+02 | global_load_balancing_loss: 9.996039E-01 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11161.047 | params rms: 0.060433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:56:08.036197 | finish at 2026-10-17 19:18:05 + [2026-10-16 02:25:41] iteration 1727/ 2384 | consumed samples: 55264 | elapsed time per iteration (ms): 224172.2 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 74840.6 | valid tokens/s: 74840.6 | physical tokens/s: 75582.8 | loss tokens/s: 54940.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.155382E-01 | lm-global-supervised-tokens: 1.231612E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.681945E-02 | lm-token-buffer-mib: 2.352529E+02 | global_load_balancing_loss: 9.992141E-01 | loss scale: 1.0 | grad norm: 0.220 | params norm: 11161.018 | params rms: 0.060433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:54:41.140233 | finish at 2026-10-17 19:20:22 + [2026-10-16 02:29:22] iteration 1728/ 2384 | consumed samples: 55296 | elapsed time per iteration (ms): 221536.4 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 75731.1 | valid tokens/s: 75731.1 | physical tokens/s: 77396.4 | loss tokens/s: 57579.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.259803E-01 | lm-global-supervised-tokens: 1.275605E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.066180E-02 | lm-token-buffer-mib: 2.363457E+02 | global_load_balancing_loss: 1.000201E+00 | loss scale: 1.0 | grad norm: 0.322 | params norm: 11160.988 | params rms: 0.060433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:22:07.865627 | finish at 2026-10-17 18:51:30 + [2026-10-16 02:33:06] iteration 1729/ 2384 | consumed samples: 55328 | elapsed time per iteration (ms): 223130.3 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75190.1 | valid tokens/s: 75190.1 | physical tokens/s: 75748.4 | loss tokens/s: 55273.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.528163E-01 | lm-global-supervised-tokens: 1.233313E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.578725E-02 | lm-token-buffer-mib: 2.341932E+02 | global_load_balancing_loss: 1.001747E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11160.955 | params rms: 0.060433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:35:50.344187 | finish at 2026-10-17 19:08:56 + [2026-10-16 02:37:09] iteration 1730/ 2384 | consumed samples: 55360 | elapsed time per iteration (ms): 243578.3 | throughput per GPU (TFLOP/s/GPU): 17.9 | MFU 5.75% | FLOPs estimator: workload-aware | logical tokens/s: 68878.1 | valid tokens/s: 68878.1 | physical tokens/s: 69478.5 | loss tokens/s: 53812.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.719870E-01 | lm-global-supervised-tokens: 1.310759E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.592072E-02 | lm-token-buffer-mib: 2.351917E+02 | global_load_balancing_loss: 1.000140E+00 | loss scale: 1.0 | grad norm: 0.267 | params norm: 11160.921 | params rms: 0.060433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:15:00.224884 | finish at 2026-10-17 22:52:09 + [2026-10-16 02:40:52] iteration 1731/ 2384 | consumed samples: 55392 | elapsed time per iteration (ms): 222646.7 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 75353.4 | valid tokens/s: 75353.4 | physical tokens/s: 76182.8 | loss tokens/s: 58799.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.709157E-01 | lm-global-supervised-tokens: 1.309148E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.908706E-02 | lm-token-buffer-mib: 2.345879E+02 | global_load_balancing_loss: 1.002055E+00 | loss scale: 1.0 | grad norm: 0.196 | params norm: 11160.887 | params rms: 0.060433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:23:08.267637 | finish at 2026-10-17 19:04:00 + [2026-10-16 02:44:36] iteration 1732/ 2384 | consumed samples: 55424 | elapsed time per iteration (ms): 223917.6 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 74925.8 | valid tokens/s: 74925.8 | physical tokens/s: 75728.4 | loss tokens/s: 60791.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.505582E-01 | lm-global-supervised-tokens: 1.361219E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.088487E-02 | lm-token-buffer-mib: 2.344486E+02 | global_load_balancing_loss: 1.002150E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11160.857 | params rms: 0.060432 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:33:14.249807 | finish at 2026-10-17 19:17:50 + [2026-10-16 02:48:19] iteration 1733/ 2384 | consumed samples: 55456 | elapsed time per iteration (ms): 223255.3 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 75148.1 | valid tokens/s: 75148.1 | physical tokens/s: 75476.4 | loss tokens/s: 56336.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.522666E-01 | lm-global-supervised-tokens: 1.257750E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.219016E-02 | lm-token-buffer-mib: 2.330097E+02 | global_load_balancing_loss: 1.001081E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11160.826 | params rms: 0.060432 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:22:19.171460 | finish at 2026-10-17 19:10:38 + [2026-10-16 02:52:04] iteration 1734/ 2384 | consumed samples: 55488 | elapsed time per iteration (ms): 224755.2 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 74646.5 | valid tokens/s: 74646.5 | physical tokens/s: 75445.1 | loss tokens/s: 56044.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.373978E-01 | lm-global-supervised-tokens: 1.259631E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.697266E-02 | lm-token-buffer-mib: 2.329830E+02 | global_load_balancing_loss: 9.990501E-01 | loss scale: 1.0 | grad norm: 0.232 | params norm: 11160.792 | params rms: 0.060432 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:34:50.873587 | finish at 2026-10-17 19:26:55 + [2026-10-16 02:55:45] iteration 1735/ 2384 | consumed samples: 55520 | elapsed time per iteration (ms): 221587.5 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.64% | FLOPs estimator: workload-aware | logical tokens/s: 75713.6 | valid tokens/s: 75713.6 | physical tokens/s: 76078.8 | loss tokens/s: 55498.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.552482E-01 | lm-global-supervised-tokens: 1.229770E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.355381E-02 | lm-token-buffer-mib: 2.327801E+02 | global_load_balancing_loss: 9.990773E-01 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11160.756 | params rms: 0.060432 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:56:50.281527 | finish at 2026-10-17 18:52:36 + [2026-10-16 02:59:28] iteration 1736/ 2384 | consumed samples: 55552 | elapsed time per iteration (ms): 222297.0 | throughput per GPU (TFLOP/s/GPU): 26.1 | MFU 8.37% | FLOPs estimator: workload-aware | logical tokens/s: 75471.8 | valid tokens/s: 75471.8 | physical tokens/s: 76507.9 | loss tokens/s: 53591.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.454513E-01 | lm-global-supervised-tokens: 1.191328E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.199892E-02 | lm-token-buffer-mib: 2.325740E+02 | global_load_balancing_loss: 1.000181E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11160.721 | params rms: 0.060432 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:00:48.486700 | finish at 2026-10-17 19:00:16 + [2026-10-16 03:03:10] iteration 1737/ 2384 | consumed samples: 55584 | elapsed time per iteration (ms): 222127.6 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 75529.6 | valid tokens/s: 75529.6 | physical tokens/s: 76733.1 | loss tokens/s: 56213.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.950751E-01 | lm-global-supervised-tokens: 1.248660E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.218976E-02 | lm-token-buffer-mib: 2.324406E+02 | global_load_balancing_loss: 1.000355E+00 | loss scale: 1.0 | grad norm: 0.196 | params norm: 11160.684 | params rms: 0.060431 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:55:16.567197 | finish at 2026-10-17 18:58:26 + [2026-10-16 03:06:50] iteration 1738/ 2384 | consumed samples: 55616 | elapsed time per iteration (ms): 220368.3 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 76132.5 | valid tokens/s: 76132.5 | physical tokens/s: 76966.7 | loss tokens/s: 58500.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.714478E-01 | lm-global-supervised-tokens: 1.289165E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.807414E-02 | lm-token-buffer-mib: 2.342694E+02 | global_load_balancing_loss: 9.993739E-01 | loss scale: 1.0 | grad norm: 0.181 | params norm: 11160.647 | params rms: 0.060431 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:32:37.931170 | finish at 2026-10-17 18:39:28 + [2026-10-16 03:10:31] iteration 1739/ 2384 | consumed samples: 55648 | elapsed time per iteration (ms): 220699.6 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.86% | FLOPs estimator: workload-aware | logical tokens/s: 76018.0 | valid tokens/s: 76018.0 | physical tokens/s: 76471.0 | loss tokens/s: 57461.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.805227E-01 | lm-global-supervised-tokens: 1.268177E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.147379E-02 | lm-token-buffer-mib: 2.325810E+02 | global_load_balancing_loss: 9.991606E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11160.610 | params rms: 0.060431 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:32:31.217229 | finish at 2026-10-17 18:43:02 + [2026-10-16 03:14:34] iteration 1740/ 2384 | consumed samples: 55680 | elapsed time per iteration (ms): 242884.9 | throughput per GPU (TFLOP/s/GPU): 16.5 | MFU 5.28% | FLOPs estimator: workload-aware | logical tokens/s: 69074.7 | valid tokens/s: 69074.7 | physical tokens/s: 69701.4 | loss tokens/s: 52176.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.984638E-01 | lm-global-supervised-tokens: 1.267292E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.623692E-02 | lm-token-buffer-mib: 2.336938E+02 | global_load_balancing_loss: 9.997537E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11160.572 | params rms: 0.060431 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:26:57.855700 | finish at 2026-10-17 22:41:32 + [2026-10-16 03:18:16] iteration 1741/ 2384 | consumed samples: 55712 | elapsed time per iteration (ms): 222657.3 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 75349.7 | valid tokens/s: 75349.7 | physical tokens/s: 75938.3 | loss tokens/s: 58646.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.586431E-01 | lm-global-supervised-tokens: 1.305808E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.332082E-02 | lm-token-buffer-mib: 2.339619E+02 | global_load_balancing_loss: 1.001350E+00 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11160.535 | params rms: 0.060431 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:46:08.658001 | finish at 2026-10-17 19:04:25 + [2026-10-16 03:22:00] iteration 1742/ 2384 | consumed samples: 55744 | elapsed time per iteration (ms): 223411.8 | throughput per GPU (TFLOP/s/GPU): 27.3 | MFU 8.77% | FLOPs estimator: workload-aware | logical tokens/s: 75095.4 | valid tokens/s: 75095.4 | physical tokens/s: 75788.2 | loss tokens/s: 54601.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.642748E-01 | lm-global-supervised-tokens: 1.219863E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.055587E-02 | lm-token-buffer-mib: 2.337624E+02 | global_load_balancing_loss: 1.003747E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11160.497 | params rms: 0.060430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:50:30.379214 | finish at 2026-10-17 19:12:30 + [2026-10-16 03:25:46] iteration 1743/ 2384 | consumed samples: 55776 | elapsed time per iteration (ms): 225841.6 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 74287.4 | valid tokens/s: 74287.4 | physical tokens/s: 75433.8 | loss tokens/s: 58936.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.556256E-01 | lm-global-supervised-tokens: 1.331020E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.320241E-02 | lm-token-buffer-mib: 2.378097E+02 | global_load_balancing_loss: 1.001445E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11160.461 | params rms: 0.060430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:12:44.483596 | finish at 2026-10-17 19:38:30 + [2026-10-16 03:29:27] iteration 1744/ 2384 | consumed samples: 55808 | elapsed time per iteration (ms): 221516.2 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.75% | FLOPs estimator: workload-aware | logical tokens/s: 75738.0 | valid tokens/s: 75738.0 | physical tokens/s: 76926.1 | loss tokens/s: 55674.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.394021E-01 | lm-global-supervised-tokens: 1.233288E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.725981E-02 | lm-token-buffer-mib: 2.329971E+02 | global_load_balancing_loss: 1.001336E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11160.428 | params rms: 0.060430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:22:50.354767 | finish at 2026-10-17 18:52:18 + [2026-10-16 03:33:12] iteration 1745/ 2384 | consumed samples: 55840 | elapsed time per iteration (ms): 225088.7 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 74536.0 | valid tokens/s: 74536.0 | physical tokens/s: 75419.7 | loss tokens/s: 57842.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.458302E-01 | lm-global-supervised-tokens: 1.301969E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.316720E-02 | lm-token-buffer-mib: 2.348785E+02 | global_load_balancing_loss: 1.000788E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11160.392 | params rms: 0.060430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:57:11.678422 | finish at 2026-10-17 19:30:24 + [2026-10-16 03:36:57] iteration 1746/ 2384 | consumed samples: 55872 | elapsed time per iteration (ms): 224748.4 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 74648.8 | valid tokens/s: 74648.8 | physical tokens/s: 75149.5 | loss tokens/s: 56854.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.739823E-01 | lm-global-supervised-tokens: 1.277803E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.699254E-02 | lm-token-buffer-mib: 2.336664E+02 | global_load_balancing_loss: 9.996288E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11160.358 | params rms: 0.060430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:49:49.465645 | finish at 2026-10-17 19:26:46 + [2026-10-16 03:40:42] iteration 1747/ 2384 | consumed samples: 55904 | elapsed time per iteration (ms): 225508.0 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.26% | FLOPs estimator: workload-aware | logical tokens/s: 74397.3 | valid tokens/s: 74397.3 | physical tokens/s: 74956.0 | loss tokens/s: 58067.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.910448E-01 | lm-global-supervised-tokens: 1.309470E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.914577E-02 | lm-token-buffer-mib: 2.342965E+02 | global_load_balancing_loss: 9.999866E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11160.325 | params rms: 0.060430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:54:08.607021 | finish at 2026-10-17 19:34:51 + [2026-10-16 03:44:27] iteration 1748/ 2384 | consumed samples: 55936 | elapsed time per iteration (ms): 224331.6 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 74787.5 | valid tokens/s: 74787.5 | physical tokens/s: 75813.2 | loss tokens/s: 60355.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.577691E-01 | lm-global-supervised-tokens: 1.353955E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.538402E-02 | lm-token-buffer-mib: 2.355454E+02 | global_load_balancing_loss: 1.000068E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11160.293 | params rms: 0.060429 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:37:54.871943 | finish at 2026-10-17 19:22:22 + [2026-10-16 03:48:07] iteration 1749/ 2384 | consumed samples: 55968 | elapsed time per iteration (ms): 220240.5 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 76176.7 | valid tokens/s: 76176.7 | physical tokens/s: 77049.4 | loss tokens/s: 54680.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.985059E-01 | lm-global-supervised-tokens: 1.204283E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.817897E-02 | lm-token-buffer-mib: 2.327614E+02 | global_load_balancing_loss: 1.000385E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11160.260 | params rms: 0.060429 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:50:52.733380 | finish at 2026-10-17 18:39:00 + [2026-10-16 03:52:14] iteration 1750/ 2384 | consumed samples: 56000 | elapsed time per iteration (ms): 246587.1 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 68037.5 | valid tokens/s: 68037.5 | physical tokens/s: 68672.8 | loss tokens/s: 53323.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.721751E-01 | lm-global-supervised-tokens: 1.314884E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.812264E-02 | lm-token-buffer-mib: 2.339222E+02 | global_load_balancing_loss: 1.000643E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11160.225 | params rms: 0.060429 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:25:36.220209 | finish at 2026-10-17 23:17:50 + [2026-10-16 03:55:56] iteration 1751/ 2384 | consumed samples: 56032 | elapsed time per iteration (ms): 222547.3 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 75387.1 | valid tokens/s: 75387.1 | physical tokens/s: 76042.9 | loss tokens/s: 59226.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.698844E-01 | lm-global-supervised-tokens: 1.318061E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.843525E-02 | lm-token-buffer-mib: 2.349700E+02 | global_load_balancing_loss: 1.000946E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11160.190 | params rms: 0.060429 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:07:52.421948 | finish at 2026-10-17 19:03:49 + [2026-10-16 03:59:39] iteration 1752/ 2384 | consumed samples: 56064 | elapsed time per iteration (ms): 222355.2 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 75452.2 | valid tokens/s: 75452.2 | physical tokens/s: 76188.3 | loss tokens/s: 57088.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.862375E-01 | lm-global-supervised-tokens: 1.269393E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.780469E-02 | lm-token-buffer-mib: 2.346690E+02 | global_load_balancing_loss: 1.000838E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11160.158 | params rms: 0.060429 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:02:08.465940 | finish at 2026-10-17 19:01:47 + [2026-10-16 04:03:18] iteration 1753/ 2384 | consumed samples: 56096 | elapsed time per iteration (ms): 219251.7 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.81% | FLOPs estimator: workload-aware | logical tokens/s: 76520.3 | valid tokens/s: 76520.3 | physical tokens/s: 76827.7 | loss tokens/s: 56009.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.305588E-01 | lm-global-supervised-tokens: 1.228021E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.736629E-02 | lm-token-buffer-mib: 2.325584E+02 | global_load_balancing_loss: 9.995115E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11160.127 | params rms: 0.060428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:25:47.830776 | finish at 2026-10-17 18:29:06 + [2026-10-16 04:07:01] iteration 1754/ 2384 | consumed samples: 56128 | elapsed time per iteration (ms): 222966.3 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75245.4 | valid tokens/s: 75245.3 | physical tokens/s: 75645.1 | loss tokens/s: 52034.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.263953E-01 | lm-global-supervised-tokens: 1.160200E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.731764E-02 | lm-token-buffer-mib: 2.327996E+02 | global_load_balancing_loss: 9.996848E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11160.096 | params rms: 0.060428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:01:08.797245 | finish at 2026-10-17 19:08:10 + [2026-10-16 04:10:40] iteration 1755/ 2384 | consumed samples: 56160 | elapsed time per iteration (ms): 219193.4 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 76540.7 | valid tokens/s: 76540.7 | physical tokens/s: 77172.8 | loss tokens/s: 56402.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.494279E-01 | lm-global-supervised-tokens: 1.236297E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.754294E-02 | lm-token-buffer-mib: 2.334304E+02 | global_load_balancing_loss: 1.001399E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11160.063 | params rms: 0.060428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:17:52.651390 | finish at 2026-10-17 18:28:33 + [2026-10-16 04:14:24] iteration 1756/ 2384 | consumed samples: 56192 | elapsed time per iteration (ms): 223663.2 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 75011.1 | valid tokens/s: 75011.1 | physical tokens/s: 76556.0 | loss tokens/s: 56803.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.972003E-01 | lm-global-supervised-tokens: 1.270475E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.627078E-02 | lm-token-buffer-mib: 2.399167E+02 | global_load_balancing_loss: 1.000872E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11160.032 | params rms: 0.060428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:01:00.464983 | finish at 2026-10-17 19:15:24 + [2026-10-16 04:18:03] iteration 1757/ 2384 | consumed samples: 56224 | elapsed time per iteration (ms): 219799.4 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 76329.6 | valid tokens/s: 76329.6 | physical tokens/s: 76748.1 | loss tokens/s: 57008.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.941284E-01 | lm-global-supervised-tokens: 1.253046E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.700696E-02 | lm-token-buffer-mib: 2.324641E+02 | global_load_balancing_loss: 1.000417E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11159.999 | params rms: 0.060428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:16:54.254652 | finish at 2026-10-17 18:34:58 + [2026-10-16 04:21:47] iteration 1758/ 2384 | consumed samples: 56256 | elapsed time per iteration (ms): 223241.1 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 75152.9 | valid tokens/s: 75152.9 | physical tokens/s: 76030.7 | loss tokens/s: 54024.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.922889E-01 | lm-global-supervised-tokens: 1.206058E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.624746E-02 | lm-token-buffer-mib: 2.345919E+02 | global_load_balancing_loss: 1.001096E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11159.967 | params rms: 0.060428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:49:08.898049 | finish at 2026-10-17 19:10:56 + [2026-10-16 04:25:29] iteration 1759/ 2384 | consumed samples: 56288 | elapsed time per iteration (ms): 222652.6 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.86% | FLOPs estimator: workload-aware | logical tokens/s: 75351.5 | valid tokens/s: 75351.5 | physical tokens/s: 76469.7 | loss tokens/s: 59438.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.816060E-01 | lm-global-supervised-tokens: 1.323420E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.614978E-02 | lm-token-buffer-mib: 2.349044E+02 | global_load_balancing_loss: 1.000021E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11159.934 | params rms: 0.060427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:39:17.859236 | finish at 2026-10-17 19:04:47 + [2026-10-16 04:29:41] iteration 1760/ 2384 | consumed samples: 56320 | elapsed time per iteration (ms): 251870.2 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 66610.5 | valid tokens/s: 66610.5 | physical tokens/s: 67479.3 | loss tokens/s: 49834.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.810386E-01 | lm-global-supervised-tokens: 1.255171E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.836742E-02 | lm-token-buffer-mib: 2.360614E+02 | global_load_balancing_loss: 1.001379E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11159.902 | params rms: 0.060427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:39:27.028255 | finish at 2026-10-18 00:09:08 +times across ranks (ms): + save-checkpoint: + rank 0: 24906.59 + rank 1: 24906.60 + rank 2: 24906.61 + rank 3: 24906.63 + rank 4: 24906.59 + rank 5: 24906.61 + rank 6: 24906.61 + rank 7: 24906.60 + rank 8: 24906.76 + rank 9: 24906.78 + rank 10: 24906.79 + rank 11: 24906.75 + rank 12: 24906.78 + rank 13: 24906.75 + rank 14: 24906.78 + rank 15: 24906.75 + rank 16: 24906.52 + rank 17: 24906.52 + rank 18: 24906.52 + rank 19: 24906.51 + rank 20: 24906.34 + rank 21: 24906.46 + rank 22: 24906.50 + rank 23: 24906.46 + rank 24: 24906.88 + rank 25: 24906.84 + rank 26: 24906.93 + rank 27: 24906.86 + rank 28: 24906.82 + rank 29: 24907.20 + rank 30: 24906.88 + rank 31: 24906.85 + rank 32: 24907.00 + rank 33: 24907.05 + rank 34: 24907.08 + rank 35: 24907.01 + rank 36: 24907.02 + rank 37: 24907.05 + rank 38: 24906.98 + rank 39: 24906.88 + rank 40: 24907.13 + rank 41: 24907.12 + rank 42: 24907.13 + rank 43: 24907.12 + rank 44: 24907.12 + rank 45: 24907.14 + rank 46: 24907.12 + rank 47: 24907.12 + rank 48: 24906.88 + rank 49: 24906.80 + rank 50: 24906.77 + rank 51: 24906.83 + rank 52: 24906.87 + rank 53: 24906.95 + rank 54: 24906.74 + rank 55: 24906.88 + rank 56: 24906.25 + rank 57: 24906.28 + rank 58: 24906.32 + rank 59: 24906.27 + rank 60: 24906.29 + rank 61: 24906.26 + rank 62: 24906.23 + rank 63: 24906.23 + [2026-10-16 04:34:01] iteration 1761/ 2384 | consumed samples: 56352 | elapsed time per iteration (ms): 233080.1 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 71980.4 | valid tokens/s: 71980.4 | physical tokens/s: 72786.9 | loss tokens/s: 53434.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.770596E-01 | lm-global-supervised-tokens: 1.245445E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.981804E-02 | lm-token-buffer-mib: 2.362395E+02 | global_load_balancing_loss: 1.000830E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11159.870 | params rms: 0.060427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:20:08.899812 | finish at 2026-10-17 20:54:10 + [2026-10-16 04:37:49] iteration 1762/ 2384 | consumed samples: 56384 | elapsed time per iteration (ms): 227497.5 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 73746.8 | valid tokens/s: 73746.8 | physical tokens/s: 74674.3 | loss tokens/s: 54710.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.767282E-01 | lm-global-supervised-tokens: 1.244660E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.522990E-02 | lm-token-buffer-mib: 2.355984E+02 | global_load_balancing_loss: 1.000082E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11159.839 | params rms: 0.060427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:18:23.453566 | finish at 2026-10-17 19:56:12 + [2026-10-16 04:41:29] iteration 1763/ 2384 | consumed samples: 56416 | elapsed time per iteration (ms): 220346.9 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 76139.9 | valid tokens/s: 76139.9 | physical tokens/s: 77336.4 | loss tokens/s: 57636.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.465494E-01 | lm-global-supervised-tokens: 1.270012E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.797961E-02 | lm-token-buffer-mib: 2.352076E+02 | global_load_balancing_loss: 1.001159E+00 | loss scale: 1.0 | grad norm: 0.264 | params norm: 11159.808 | params rms: 0.060427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:00:35.413491 | finish at 2026-10-17 18:42:05 + [2026-10-16 04:45:10] iteration 1764/ 2384 | consumed samples: 56448 | elapsed time per iteration (ms): 220826.8 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.40% | FLOPs estimator: workload-aware | logical tokens/s: 75974.5 | valid tokens/s: 75974.5 | physical tokens/s: 76500.9 | loss tokens/s: 57453.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.823555E-01 | lm-global-supervised-tokens: 1.268737E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.359430E-02 | lm-token-buffer-mib: 2.327021E+02 | global_load_balancing_loss: 1.000624E+00 | loss scale: 1.0 | grad norm: 0.253 | params norm: 11159.780 | params rms: 0.060427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:01:52.619910 | finish at 2026-10-17 18:47:03 + [2026-10-16 04:48:55] iteration 1765/ 2384 | consumed samples: 56480 | elapsed time per iteration (ms): 225056.7 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.65% | FLOPs estimator: workload-aware | logical tokens/s: 74546.6 | valid tokens/s: 74546.6 | physical tokens/s: 75306.8 | loss tokens/s: 52826.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.079619E-01 | lm-global-supervised-tokens: 1.188885E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.033056E-02 | lm-token-buffer-mib: 2.335401E+02 | global_load_balancing_loss: 1.000974E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11159.752 | params rms: 0.060426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:41:50.069254 | finish at 2026-10-17 19:30:45 + [2026-10-16 04:52:36] iteration 1766/ 2384 | consumed samples: 56512 | elapsed time per iteration (ms): 221336.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75799.6 | valid tokens/s: 75799.6 | physical tokens/s: 76641.2 | loss tokens/s: 59688.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.200585E-01 | lm-global-supervised-tokens: 1.321116E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.644812E-02 | lm-token-buffer-mib: 2.331931E+02 | global_load_balancing_loss: 1.000746E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11159.726 | params rms: 0.060426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:59:45.665218 | finish at 2026-10-17 18:52:22 + [2026-10-16 04:56:19] iteration 1767/ 2384 | consumed samples: 56544 | elapsed time per iteration (ms): 222913.7 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 75263.2 | valid tokens/s: 75263.2 | physical tokens/s: 75887.2 | loss tokens/s: 58681.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.251661E-01 | lm-global-supervised-tokens: 1.308090E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.790982E-02 | lm-token-buffer-mib: 2.351360E+02 | global_load_balancing_loss: 1.000018E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11159.694 | params rms: 0.060426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:12:17.747374 | finish at 2026-10-17 19:08:37 + [2026-10-16 05:00:03] iteration 1768/ 2384 | consumed samples: 56576 | elapsed time per iteration (ms): 223743.3 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 74984.1 | valid tokens/s: 74984.1 | physical tokens/s: 75330.9 | loss tokens/s: 54662.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.759908E-01 | lm-global-supervised-tokens: 1.223026E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.163900E-02 | lm-token-buffer-mib: 2.335345E+02 | global_load_balancing_loss: 1.000377E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11159.665 | params rms: 0.060426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:17:05.859852 | finish at 2026-10-17 19:17:09 + [2026-10-16 05:03:46] iteration 1769/ 2384 | consumed samples: 56608 | elapsed time per iteration (ms): 223144.0 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 75185.5 | valid tokens/s: 75185.5 | physical tokens/s: 75451.7 | loss tokens/s: 56436.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.771938E-01 | lm-global-supervised-tokens: 1.259348E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.499096E-02 | lm-token-buffer-mib: 2.330449E+02 | global_load_balancing_loss: 1.000629E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11159.636 | params rms: 0.060426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:07:13.585546 | finish at 2026-10-17 19:11:00 + [2026-10-16 05:07:50] iteration 1770/ 2384 | consumed samples: 56640 | elapsed time per iteration (ms): 243272.9 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 68964.4 | valid tokens/s: 68964.4 | physical tokens/s: 69436.8 | loss tokens/s: 53068.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.329621E-01 | lm-global-supervised-tokens: 1.291011E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.263231E-02 | lm-token-buffer-mib: 2.340332E+02 | global_load_balancing_loss: 9.998935E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11159.608 | params rms: 0.060426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:29:29.544269 | finish at 2026-10-17 22:37:19 + [2026-10-16 05:11:35] iteration 1771/ 2384 | consumed samples: 56672 | elapsed time per iteration (ms): 225494.2 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 74401.9 | valid tokens/s: 74401.9 | physical tokens/s: 74898.9 | loss tokens/s: 56343.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.497231E-01 | lm-global-supervised-tokens: 1.270522E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.587762E-02 | lm-token-buffer-mib: 2.341190E+02 | global_load_balancing_loss: 1.000188E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11159.583 | params rms: 0.060426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:23:47.974994 | finish at 2026-10-17 19:35:23 + [2026-10-16 05:15:21] iteration 1772/ 2384 | consumed samples: 56704 | elapsed time per iteration (ms): 226417.8 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.90% | FLOPs estimator: workload-aware | logical tokens/s: 74098.3 | valid tokens/s: 74098.3 | physical tokens/s: 74641.5 | loss tokens/s: 55941.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.891490E-01 | lm-global-supervised-tokens: 1.266617E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.424525E-02 | lm-token-buffer-mib: 2.341192E+02 | global_load_balancing_loss: 1.000572E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11159.554 | params rms: 0.060425 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:29:27.701594 | finish at 2026-10-17 19:44:49 + [2026-10-16 05:19:13] iteration 1773/ 2384 | consumed samples: 56736 | elapsed time per iteration (ms): 231326.6 | throughput per GPU (TFLOP/s/GPU): 18.5 | MFU 5.93% | FLOPs estimator: workload-aware | logical tokens/s: 72526.0 | valid tokens/s: 72526.0 | physical tokens/s: 74188.6 | loss tokens/s: 55103.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.867308E-01 | lm-global-supervised-tokens: 1.274679E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.694676E-02 | lm-token-buffer-mib: 2.392592E+02 | global_load_balancing_loss: 1.000556E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11159.521 | params rms: 0.060425 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:15:40.537641 | finish at 2026-10-17 20:34:53 + [2026-10-16 05:22:54] iteration 1774/ 2384 | consumed samples: 56768 | elapsed time per iteration (ms): 221489.4 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.88% | FLOPs estimator: workload-aware | logical tokens/s: 75747.1 | valid tokens/s: 75747.1 | physical tokens/s: 76182.1 | loss tokens/s: 54211.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.711266E-01 | lm-global-supervised-tokens: 1.200716E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.434084E-02 | lm-token-buffer-mib: 2.343374E+02 | global_load_balancing_loss: 1.001498E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11159.491 | params rms: 0.060425 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:31:48.535254 | finish at 2026-10-17 18:54:43 + [2026-10-16 05:26:35] iteration 1775/ 2384 | consumed samples: 56800 | elapsed time per iteration (ms): 220470.8 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 76097.1 | valid tokens/s: 76097.1 | physical tokens/s: 76596.2 | loss tokens/s: 56229.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.667443E-01 | lm-global-supervised-tokens: 1.239691E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.573054E-02 | lm-token-buffer-mib: 2.330948E+02 | global_load_balancing_loss: 9.997556E-01 | loss scale: 1.0 | grad norm: 0.284 | params norm: 11159.459 | params rms: 0.060425 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:17:46.734141 | finish at 2026-10-17 18:44:21 + [2026-10-16 05:30:15] iteration 1776/ 2384 | consumed samples: 56832 | elapsed time per iteration (ms): 220584.0 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 76058.0 | valid tokens/s: 76058.0 | physical tokens/s: 76604.5 | loss tokens/s: 55014.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.588652E-01 | lm-global-supervised-tokens: 1.213528E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.771986E-02 | lm-token-buffer-mib: 2.332832E+02 | global_load_balancing_loss: 1.000023E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11159.426 | params rms: 0.060425 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:15:15.089172 | finish at 2026-10-17 18:45:30 + [2026-10-16 05:33:58] iteration 1777/ 2384 | consumed samples: 56864 | elapsed time per iteration (ms): 222720.3 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 75328.5 | valid tokens/s: 75328.5 | physical tokens/s: 75950.8 | loss tokens/s: 56356.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.175919E-01 | lm-global-supervised-tokens: 1.255167E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.451174E-02 | lm-token-buffer-mib: 2.343119E+02 | global_load_balancing_loss: 9.994726E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11159.393 | params rms: 0.060424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:33:11.229601 | finish at 2026-10-17 19:07:09 + [2026-10-16 05:37:40] iteration 1778/ 2384 | consumed samples: 56896 | elapsed time per iteration (ms): 222173.4 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75513.8 | valid tokens/s: 75513.8 | physical tokens/s: 76123.0 | loss tokens/s: 57328.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.930365E-01 | lm-global-supervised-tokens: 1.273693E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.762814E-02 | lm-token-buffer-mib: 2.341957E+02 | global_load_balancing_loss: 9.999455E-01 | loss scale: 1.0 | grad norm: 0.358 | params norm: 11159.360 | params rms: 0.060424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:23:57.069952 | finish at 2026-10-17 19:01:37 + [2026-10-16 05:41:20] iteration 1779/ 2384 | consumed samples: 56928 | elapsed time per iteration (ms): 219841.1 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 76315.2 | valid tokens/s: 76315.2 | physical tokens/s: 77558.9 | loss tokens/s: 59991.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.224168E-01 | lm-global-supervised-tokens: 1.318859E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.289661E-02 | lm-token-buffer-mib: 2.343242E+02 | global_load_balancing_loss: 9.999493E-01 | loss scale: 1.0 | grad norm: 0.317 | params norm: 11159.330 | params rms: 0.060424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:56:43.839955 | finish at 2026-10-17 18:38:04 + [2026-10-16 05:45:23] iteration 1780/ 2384 | consumed samples: 56960 | elapsed time per iteration (ms): 243255.4 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.26% | FLOPs estimator: workload-aware | logical tokens/s: 68969.6 | valid tokens/s: 68969.6 | physical tokens/s: 69781.7 | loss tokens/s: 49954.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.796114E-01 | lm-global-supervised-tokens: 1.215162E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.545360E-02 | lm-token-buffer-mib: 2.367484E+02 | global_load_balancing_loss: 1.000538E+00 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11159.303 | params rms: 0.060424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:48:46.236796 | finish at 2026-10-17 22:34:10 + [2026-10-16 05:49:07] iteration 1781/ 2384 | consumed samples: 56992 | elapsed time per iteration (ms): 224068.3 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 74875.3 | valid tokens/s: 74875.3 | physical tokens/s: 75572.1 | loss tokens/s: 60516.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.153746E-01 | lm-global-supervised-tokens: 1.355989E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.382738E-02 | lm-token-buffer-mib: 2.336788E+02 | global_load_balancing_loss: 1.001354E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11159.276 | params rms: 0.060424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:31:53.192237 | finish at 2026-10-17 19:21:01 + [2026-10-16 05:52:48] iteration 1782/ 2384 | consumed samples: 57024 | elapsed time per iteration (ms): 220661.5 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 76031.4 | valid tokens/s: 76031.4 | physical tokens/s: 76929.1 | loss tokens/s: 54031.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.792645E-01 | lm-global-supervised-tokens: 1.192257E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.804980E-02 | lm-token-buffer-mib: 2.360217E+02 | global_load_balancing_loss: 1.001569E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11159.248 | params rms: 0.060424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:53:58.234612 | finish at 2026-10-17 18:46:46 + [2026-10-16 05:56:33] iteration 1783/ 2384 | consumed samples: 57056 | elapsed time per iteration (ms): 225026.2 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.63% | FLOPs estimator: workload-aware | logical tokens/s: 74556.7 | valid tokens/s: 74556.7 | physical tokens/s: 75321.5 | loss tokens/s: 54947.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.551754E-01 | lm-global-supervised-tokens: 1.236465E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.358968E-02 | lm-token-buffer-mib: 2.335970E+02 | global_load_balancing_loss: 1.001930E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11159.219 | params rms: 0.060424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:34:00.751965 | finish at 2026-10-17 19:30:34 + [2026-10-16 06:00:19] iteration 1784/ 2384 | consumed samples: 57088 | elapsed time per iteration (ms): 226229.5 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 74160.0 | valid tokens/s: 74160.0 | physical tokens/s: 74638.6 | loss tokens/s: 53235.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.050870E-01 | lm-global-supervised-tokens: 1.204348E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.952971E-02 | lm-token-buffer-mib: 2.329543E+02 | global_load_balancing_loss: 1.000838E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11159.190 | params rms: 0.060423 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:42:17.674284 | finish at 2026-10-17 19:42:37 + [2026-10-16 06:04:04] iteration 1785/ 2384 | consumed samples: 57120 | elapsed time per iteration (ms): 225063.8 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 74544.0 | valid tokens/s: 74544.0 | physical tokens/s: 75098.2 | loss tokens/s: 56959.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.969489E-01 | lm-global-supervised-tokens: 1.281959E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.734008E-02 | lm-token-buffer-mib: 2.335003E+02 | global_load_balancing_loss: 1.000629E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11159.160 | params rms: 0.060423 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:26:53.223398 | finish at 2026-10-17 19:30:58 + [2026-10-16 06:07:51] iteration 1786/ 2384 | consumed samples: 57152 | elapsed time per iteration (ms): 226583.4 | throughput per GPU (TFLOP/s/GPU): 25.3 | MFU 8.11% | FLOPs estimator: workload-aware | logical tokens/s: 74044.1 | valid tokens/s: 74044.1 | physical tokens/s: 74255.6 | loss tokens/s: 56614.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.115286E-01 | lm-global-supervised-tokens: 1.282795E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.344609E-02 | lm-token-buffer-mib: 2.330335E+02 | global_load_balancing_loss: 1.000938E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11159.129 | params rms: 0.060423 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:38:16.870070 | finish at 2026-10-17 19:46:08 + [2026-10-16 06:11:33] iteration 1787/ 2384 | consumed samples: 57184 | elapsed time per iteration (ms): 222233.1 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 75493.7 | valid tokens/s: 75493.7 | physical tokens/s: 76290.1 | loss tokens/s: 51502.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.786922E-01 | lm-global-supervised-tokens: 1.144561E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.894261E-02 | lm-token-buffer-mib: 2.347183E+02 | global_load_balancing_loss: 9.991002E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11159.093 | params rms: 0.060423 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:51:13.156250 | finish at 2026-10-17 19:02:46 + [2026-10-16 06:15:18] iteration 1788/ 2384 | consumed samples: 57216 | elapsed time per iteration (ms): 225151.8 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 74515.1 | valid tokens/s: 74515.1 | physical tokens/s: 75384.4 | loss tokens/s: 56257.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.059970E-01 | lm-global-supervised-tokens: 1.266646E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.907740E-02 | lm-token-buffer-mib: 2.346846E+02 | global_load_balancing_loss: 9.985527E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11159.056 | params rms: 0.060423 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:16:30.488666 | finish at 2026-10-17 19:31:49 + [2026-10-16 06:18:59] iteration 1789/ 2384 | consumed samples: 57248 | elapsed time per iteration (ms): 220476.5 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 76094.9 | valid tokens/s: 76094.9 | physical tokens/s: 76541.7 | loss tokens/s: 57556.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.519226E-01 | lm-global-supervised-tokens: 1.268994E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.515376E-02 | lm-token-buffer-mib: 2.331395E+02 | global_load_balancing_loss: 9.985766E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11159.020 | params rms: 0.060422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:26:23.535111 | finish at 2026-10-17 18:45:22 + [2026-10-16 06:23:02] iteration 1790/ 2384 | consumed samples: 57280 | elapsed time per iteration (ms): 243046.7 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 5.99% | FLOPs estimator: workload-aware | logical tokens/s: 69028.7 | valid tokens/s: 69028.7 | physical tokens/s: 69547.8 | loss tokens/s: 50985.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.648144E-01 | lm-global-supervised-tokens: 1.239189E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.060932E-02 | lm-token-buffer-mib: 2.328455E+02 | global_load_balancing_loss: 1.000088E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11158.982 | params rms: 0.060422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:06:09.714592 | finish at 2026-10-17 22:29:12 + [2026-10-16 06:26:45] iteration 1791/ 2384 | consumed samples: 57312 | elapsed time per iteration (ms): 223197.0 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 75167.7 | valid tokens/s: 75167.7 | physical tokens/s: 76151.9 | loss tokens/s: 60834.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.682294E-01 | lm-global-supervised-tokens: 1.357812E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.701396E-02 | lm-token-buffer-mib: 2.362854E+02 | global_load_balancing_loss: 1.000804E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11158.945 | params rms: 0.060422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:45:55.811260 | finish at 2026-10-17 19:12:41 + [2026-10-16 06:30:28] iteration 1792/ 2384 | consumed samples: 57344 | elapsed time per iteration (ms): 223083.2 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 75205.9 | valid tokens/s: 75205.9 | physical tokens/s: 76189.0 | loss tokens/s: 54679.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.841559E-01 | lm-global-supervised-tokens: 1.219805E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.751603E-02 | lm-token-buffer-mib: 2.348583E+02 | global_load_balancing_loss: 1.002621E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11158.909 | params rms: 0.060422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:41:05.244083 | finish at 2026-10-17 19:11:33 + [2026-10-16 06:34:10] iteration 1793/ 2384 | consumed samples: 57376 | elapsed time per iteration (ms): 221652.5 | throughput per GPU (TFLOP/s/GPU): 25.5 | MFU 8.18% | FLOPs estimator: workload-aware | logical tokens/s: 75691.2 | valid tokens/s: 75691.2 | physical tokens/s: 76746.3 | loss tokens/s: 53718.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.093213E-01 | lm-global-supervised-tokens: 1.190680E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.696758E-02 | lm-token-buffer-mib: 2.333361E+02 | global_load_balancing_loss: 1.002305E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11158.877 | params rms: 0.060422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:23:16.598141 | finish at 2026-10-17 18:57:26 + [2026-10-16 06:37:49] iteration 1794/ 2384 | consumed samples: 57408 | elapsed time per iteration (ms): 219039.6 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.28% | FLOPs estimator: workload-aware | logical tokens/s: 76594.4 | valid tokens/s: 76594.4 | physical tokens/s: 77243.1 | loss tokens/s: 57729.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.605546E-01 | lm-global-supervised-tokens: 1.264507E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.504330E-02 | lm-token-buffer-mib: 2.325859E+02 | global_load_balancing_loss: 1.001263E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11158.843 | params rms: 0.060422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:53:53.363657 | finish at 2026-10-17 18:31:42 + [2026-10-16 06:41:35] iteration 1795/ 2384 | consumed samples: 57440 | elapsed time per iteration (ms): 225889.9 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 74271.5 | valid tokens/s: 74271.5 | physical tokens/s: 75286.5 | loss tokens/s: 57274.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.139663E-01 | lm-global-supervised-tokens: 1.293782E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.490287E-02 | lm-token-buffer-mib: 2.376234E+02 | global_load_balancing_loss: 9.998565E-01 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11158.807 | params rms: 0.060421 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:57:29.132967 | finish at 2026-10-17 19:39:04 + [2026-10-16 06:45:18] iteration 1796/ 2384 | consumed samples: 57472 | elapsed time per iteration (ms): 223126.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 75191.3 | valid tokens/s: 75191.3 | physical tokens/s: 76057.1 | loss tokens/s: 54078.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.597667E-01 | lm-global-supervised-tokens: 1.206630E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.676264E-02 | lm-token-buffer-mib: 2.365973E+02 | global_load_balancing_loss: 9.998260E-01 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11158.772 | params rms: 0.060421 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:26:38.509789 | finish at 2026-10-17 19:11:56 + [2026-10-16 06:49:01] iteration 1797/ 2384 | consumed samples: 57504 | elapsed time per iteration (ms): 222963.4 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.79% | FLOPs estimator: workload-aware | logical tokens/s: 75246.2 | valid tokens/s: 75246.2 | physical tokens/s: 76306.3 | loss tokens/s: 60951.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.797194E-01 | lm-global-supervised-tokens: 1.359005E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.293620E-02 | lm-token-buffer-mib: 2.348149E+02 | global_load_balancing_loss: 9.990296E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11158.736 | params rms: 0.060421 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:21:19.524131 | finish at 2026-10-17 19:10:20 + [2026-10-16 06:52:46] iteration 1798/ 2384 | consumed samples: 57536 | elapsed time per iteration (ms): 225562.4 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.86% | FLOPs estimator: workload-aware | logical tokens/s: 74379.2 | valid tokens/s: 74379.2 | physical tokens/s: 75155.2 | loss tokens/s: 63346.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.068435E-01 | lm-global-supervised-tokens: 1.428853E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.807042E-02 | lm-token-buffer-mib: 2.343357E+02 | global_load_balancing_loss: 9.990770E-01 | loss scale: 1.0 | grad norm: 0.185 | params norm: 11158.701 | params rms: 0.060421 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:42:59.575262 | finish at 2026-10-17 19:35:46 + [2026-10-16 06:56:29] iteration 1799/ 2384 | consumed samples: 57568 | elapsed time per iteration (ms): 222413.2 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 75432.5 | valid tokens/s: 75432.5 | physical tokens/s: 75717.5 | loss tokens/s: 55786.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.106880E-01 | lm-global-supervised-tokens: 1.240765E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.469165E-02 | lm-token-buffer-mib: 2.327170E+02 | global_load_balancing_loss: 1.000145E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11158.664 | params rms: 0.060421 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:08:31.745653 | finish at 2026-10-17 19:05:01 + [2026-10-16 07:00:31] iteration 1800/ 2384 | consumed samples: 57600 | elapsed time per iteration (ms): 241922.5 | throughput per GPU (TFLOP/s/GPU): 17.5 | MFU 5.61% | FLOPs estimator: workload-aware | logical tokens/s: 69349.5 | valid tokens/s: 69349.5 | physical tokens/s: 70067.0 | loss tokens/s: 52177.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.603688E-01 | lm-global-supervised-tokens: 1.262289E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.338200E-02 | lm-token-buffer-mib: 2.341575E+02 | global_load_balancing_loss: 1.000421E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11158.626 | params rms: 0.060420 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:14:42.730192 | finish at 2026-10-17 22:15:13 +times across ranks (ms): + save-checkpoint: + rank 0: 25389.29 + rank 1: 25389.32 + rank 2: 25389.17 + rank 3: 25389.29 + rank 4: 25389.25 + rank 5: 25389.20 + rank 6: 25389.18 + rank 7: 25389.17 + rank 8: 25389.36 + rank 9: 25389.39 + rank 10: 25389.40 + rank 11: 25389.33 + rank 12: 25389.34 + rank 13: 25389.34 + rank 14: 25389.36 + rank 15: 25389.35 + rank 16: 25388.57 + rank 17: 25388.56 + rank 18: 25388.57 + rank 19: 25388.58 + rank 20: 25388.54 + rank 21: 25388.54 + rank 22: 25388.58 + rank 23: 25388.55 + rank 24: 25390.14 + rank 25: 25390.12 + rank 26: 25390.15 + rank 27: 25390.10 + rank 28: 25389.85 + rank 29: 25390.13 + rank 30: 25390.13 + rank 31: 25390.12 + rank 32: 25388.54 + rank 33: 25388.69 + rank 34: 25388.59 + rank 35: 25388.55 + rank 36: 25388.60 + rank 37: 25388.57 + rank 38: 25388.57 + rank 39: 25388.59 + rank 40: 25388.65 + rank 41: 25388.64 + rank 42: 25388.62 + rank 43: 25388.64 + rank 44: 25388.60 + rank 45: 25388.61 + rank 46: 25388.61 + rank 47: 25388.61 + rank 48: 25387.26 + rank 49: 25387.30 + rank 50: 25387.45 + rank 51: 25387.42 + rank 52: 25387.38 + rank 53: 25387.45 + rank 54: 25387.26 + rank 55: 25387.37 + rank 56: 25388.73 + rank 57: 25388.77 + rank 58: 25388.77 + rank 59: 25388.75 + rank 60: 25388.76 + rank 61: 25388.73 + rank 62: 25388.73 + rank 63: 25388.73 + [2026-10-16 07:04:39] iteration 1801/ 2384 | consumed samples: 57632 | elapsed time per iteration (ms): 222542.7 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75388.6 | valid tokens/s: 75388.6 | physical tokens/s: 76349.6 | loss tokens/s: 57759.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.053053E-01 | lm-global-supervised-tokens: 1.285390E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.735566E-02 | lm-token-buffer-mib: 2.357122E+02 | global_load_balancing_loss: 1.001113E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11158.584 | params rms: 0.060420 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:02:22.400802 | finish at 2026-10-17 19:07:01 + [2026-10-16 07:08:31] iteration 1802/ 2384 | consumed samples: 57664 | elapsed time per iteration (ms): 231782.6 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 72383.3 | valid tokens/s: 72383.3 | physical tokens/s: 73541.5 | loss tokens/s: 53112.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.064953E-01 | lm-global-supervised-tokens: 1.231049E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.749474E-02 | lm-token-buffer-mib: 2.363189E+02 | global_load_balancing_loss: 1.001667E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11158.544 | params rms: 0.060420 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:28:17.497440 | finish at 2026-10-17 20:36:48 + [2026-10-16 07:12:14] iteration 1803/ 2384 | consumed samples: 57696 | elapsed time per iteration (ms): 223336.4 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.70% | FLOPs estimator: workload-aware | logical tokens/s: 75120.8 | valid tokens/s: 75120.8 | physical tokens/s: 75502.8 | loss tokens/s: 56661.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.299214E-01 | lm-global-supervised-tokens: 1.265460E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.489656E-02 | lm-token-buffer-mib: 2.335343E+02 | global_load_balancing_loss: 1.001500E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11158.502 | params rms: 0.060420 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:02:38.476400 | finish at 2026-10-17 19:14:52 + [2026-10-16 07:15:55] iteration 1804/ 2384 | consumed samples: 57728 | elapsed time per iteration (ms): 221532.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75732.6 | valid tokens/s: 75732.6 | physical tokens/s: 76148.1 | loss tokens/s: 57572.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.818285E-01 | lm-global-supervised-tokens: 1.275407E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.769783E-02 | lm-token-buffer-mib: 2.336543E+02 | global_load_balancing_loss: 1.001094E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11158.458 | params rms: 0.060419 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:41:28.563771 | finish at 2026-10-17 18:57:24 + [2026-10-16 07:19:40] iteration 1805/ 2384 | consumed samples: 57760 | elapsed time per iteration (ms): 224123.3 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 74856.9 | valid tokens/s: 74856.9 | physical tokens/s: 75585.0 | loss tokens/s: 57940.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.291259E-01 | lm-global-supervised-tokens: 1.298580E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.140080E-02 | lm-token-buffer-mib: 2.329133E+02 | global_load_balancing_loss: 1.000162E+00 | loss scale: 1.0 | grad norm: 0.285 | params norm: 11158.414 | params rms: 0.060419 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:02:47.379424 | finish at 2026-10-17 19:22:27 + [2026-10-16 07:23:27] iteration 1806/ 2384 | consumed samples: 57792 | elapsed time per iteration (ms): 227374.6 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.70% | FLOPs estimator: workload-aware | logical tokens/s: 73786.6 | valid tokens/s: 73786.6 | physical tokens/s: 74601.5 | loss tokens/s: 50414.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.639579E-01 | lm-global-supervised-tokens: 1.146291E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.258167E-02 | lm-token-buffer-mib: 2.340376E+02 | global_load_balancing_loss: 9.995502E-01 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11158.368 | params rms: 0.060419 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:30:22.530751 | finish at 2026-10-17 19:53:49 + [2026-10-16 07:27:11] iteration 1807/ 2384 | consumed samples: 57824 | elapsed time per iteration (ms): 224360.3 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 74777.9 | valid tokens/s: 74777.9 | physical tokens/s: 75162.2 | loss tokens/s: 55921.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.840348E-01 | lm-global-supervised-tokens: 1.254647E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.888240E-02 | lm-token-buffer-mib: 2.332901E+02 | global_load_balancing_loss: 9.987783E-01 | loss scale: 1.0 | grad norm: 0.338 | params norm: 11158.320 | params rms: 0.060419 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:57:35.893137 | finish at 2026-10-17 19:24:47 + [2026-10-16 07:30:55] iteration 1808/ 2384 | consumed samples: 57856 | elapsed time per iteration (ms): 224150.7 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 74847.9 | valid tokens/s: 74847.9 | physical tokens/s: 75327.0 | loss tokens/s: 54892.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.985058E-01 | lm-global-supervised-tokens: 1.230421E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.383635E-02 | lm-token-buffer-mib: 2.341836E+02 | global_load_balancing_loss: 9.986558E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11158.277 | params rms: 0.060418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:51:50.782379 | finish at 2026-10-17 19:22:46 + [2026-10-16 07:34:42] iteration 1809/ 2384 | consumed samples: 57888 | elapsed time per iteration (ms): 226081.0 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.36% | FLOPs estimator: workload-aware | logical tokens/s: 74208.7 | valid tokens/s: 74208.7 | physical tokens/s: 74574.6 | loss tokens/s: 53458.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.803657E-01 | lm-global-supervised-tokens: 1.208584E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.330822E-02 | lm-token-buffer-mib: 2.336986E+02 | global_load_balancing_loss: 1.000546E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11158.239 | params rms: 0.060418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:06:36.584374 | finish at 2026-10-17 19:41:18 + [2026-10-16 07:38:48] iteration 1810/ 2384 | consumed samples: 57920 | elapsed time per iteration (ms): 246275.0 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.52% | FLOPs estimator: workload-aware | logical tokens/s: 68123.8 | valid tokens/s: 68123.8 | physical tokens/s: 68773.9 | loss tokens/s: 49178.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.466571E-01 | lm-global-supervised-tokens: 1.211144E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.840727E-02 | lm-token-buffer-mib: 2.355885E+02 | global_load_balancing_loss: 1.000549E+00 | loss scale: 1.0 | grad norm: 0.210 | params norm: 11158.206 | params rms: 0.060418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:16:01.849781 | finish at 2026-10-17 22:54:50 + [2026-10-16 07:42:30] iteration 1811/ 2384 | consumed samples: 57952 | elapsed time per iteration (ms): 222052.4 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75555.1 | valid tokens/s: 75555.1 | physical tokens/s: 76254.2 | loss tokens/s: 57245.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.055134E-01 | lm-global-supervised-tokens: 1.271153E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.753390E-02 | lm-token-buffer-mib: 2.332409E+02 | global_load_balancing_loss: 9.999173E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11158.173 | params rms: 0.060418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:20:36.013789 | finish at 2026-10-17 19:03:06 + [2026-10-16 07:46:15] iteration 1812/ 2384 | consumed samples: 57984 | elapsed time per iteration (ms): 225385.2 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 74437.8 | valid tokens/s: 74437.8 | physical tokens/s: 75801.0 | loss tokens/s: 60832.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.825119E-01 | lm-global-supervised-tokens: 1.371083E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.834394E-02 | lm-token-buffer-mib: 2.347272E+02 | global_load_balancing_loss: 1.000520E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11158.140 | params rms: 0.060418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:48:40.319549 | finish at 2026-10-17 19:34:56 + [2026-10-16 07:50:00] iteration 1813/ 2384 | consumed samples: 58016 | elapsed time per iteration (ms): 224714.5 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.13% | FLOPs estimator: workload-aware | logical tokens/s: 74660.1 | valid tokens/s: 74660.1 | physical tokens/s: 77024.0 | loss tokens/s: 60185.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.894625E-01 | lm-global-supervised-tokens: 1.352459E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.839310E-02 | lm-token-buffer-mib: 2.368250E+02 | global_load_balancing_loss: 1.001042E+00 | loss scale: 1.0 | grad norm: 0.194 | params norm: 11158.107 | params rms: 0.060418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:38:31.964224 | finish at 2026-10-17 19:28:32 + [2026-10-16 07:53:44] iteration 1814/ 2384 | consumed samples: 58048 | elapsed time per iteration (ms): 223844.9 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 74950.0 | valid tokens/s: 74950.0 | physical tokens/s: 75713.3 | loss tokens/s: 57854.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.649906E-01 | lm-global-supervised-tokens: 1.295052E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.327357E-02 | lm-token-buffer-mib: 2.353134E+02 | global_load_balancing_loss: 1.001453E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11158.081 | params rms: 0.060417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:26:31.583834 | finish at 2026-10-17 19:20:15 + [2026-10-16 07:57:27] iteration 1815/ 2384 | consumed samples: 58080 | elapsed time per iteration (ms): 223458.5 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.21% | FLOPs estimator: workload-aware | logical tokens/s: 75079.6 | valid tokens/s: 75079.6 | physical tokens/s: 76026.1 | loss tokens/s: 56094.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.525883E-01 | lm-global-supervised-tokens: 1.253484E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.334703E-02 | lm-token-buffer-mib: 2.335166E+02 | global_load_balancing_loss: 1.001656E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11158.058 | params rms: 0.060417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:19:07.869897 | finish at 2026-10-17 19:16:35 + [2026-10-16 08:01:10] iteration 1816/ 2384 | consumed samples: 58112 | elapsed time per iteration (ms): 223028.2 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.09% | FLOPs estimator: workload-aware | logical tokens/s: 75224.7 | valid tokens/s: 75224.6 | physical tokens/s: 76292.7 | loss tokens/s: 56753.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.547425E-01 | lm-global-supervised-tokens: 1.265766E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.528392E-02 | lm-token-buffer-mib: 2.351588E+02 | global_load_balancing_loss: 1.000387E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11158.031 | params rms: 0.060417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:11:19.991278 | finish at 2026-10-17 19:12:30 + [2026-10-16 08:04:52] iteration 1817/ 2384 | consumed samples: 58144 | elapsed time per iteration (ms): 221815.5 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75635.7 | valid tokens/s: 75635.7 | physical tokens/s: 76880.9 | loss tokens/s: 59321.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.509022E-01 | lm-global-supervised-tokens: 1.315845E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.899957E-02 | lm-token-buffer-mib: 2.358708E+02 | global_load_balancing_loss: 1.000699E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11158.005 | params rms: 0.060417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:56:09.363773 | finish at 2026-10-17 19:01:01 + [2026-10-16 08:08:36] iteration 1818/ 2384 | consumed samples: 58176 | elapsed time per iteration (ms): 223704.1 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 74997.4 | valid tokens/s: 74997.4 | physical tokens/s: 76203.8 | loss tokens/s: 55303.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.257870E-01 | lm-global-supervised-tokens: 1.237168E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.310181E-02 | lm-token-buffer-mib: 2.368946E+02 | global_load_balancing_loss: 9.997388E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11157.978 | params rms: 0.060417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:10:16.526882 | finish at 2026-10-17 19:18:52 + [2026-10-16 08:12:18] iteration 1819/ 2384 | consumed samples: 58208 | elapsed time per iteration (ms): 222323.5 | throughput per GPU (TFLOP/s/GPU): 25.4 | MFU 8.14% | FLOPs estimator: workload-aware | logical tokens/s: 75463.0 | valid tokens/s: 75463.0 | physical tokens/s: 75732.5 | loss tokens/s: 53708.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.880811E-01 | lm-global-supervised-tokens: 1.194061E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.993056E-02 | lm-token-buffer-mib: 2.330541E+02 | global_load_balancing_loss: 1.000615E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11157.951 | params rms: 0.060417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:53:32.794023 | finish at 2026-10-17 19:05:51 + [2026-10-16 08:16:21] iteration 1820/ 2384 | consumed samples: 58240 | elapsed time per iteration (ms): 243083.9 | throughput per GPU (TFLOP/s/GPU): 17.7 | MFU 5.68% | FLOPs estimator: workload-aware | logical tokens/s: 69018.2 | valid tokens/s: 69018.2 | physical tokens/s: 70180.1 | loss tokens/s: 54417.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.274380E-01 | lm-global-supervised-tokens: 1.322804E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.740591E-02 | lm-token-buffer-mib: 2.359658E+02 | global_load_balancing_loss: 9.993958E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11157.924 | params rms: 0.060417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:04:59.310173 | finish at 2026-10-17 22:21:21 + [2026-10-16 08:20:08] iteration 1821/ 2384 | consumed samples: 58272 | elapsed time per iteration (ms): 226943.4 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 73926.8 | valid tokens/s: 73926.8 | physical tokens/s: 74993.1 | loss tokens/s: 54805.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.397429E-01 | lm-global-supervised-tokens: 1.243768E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.350987E-02 | lm-token-buffer-mib: 2.381002E+02 | global_load_balancing_loss: 9.992098E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11157.894 | params rms: 0.060416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:29:29.143812 | finish at 2026-10-17 19:49:37 + [2026-10-16 08:23:50] iteration 1822/ 2384 | consumed samples: 58304 | elapsed time per iteration (ms): 221872.7 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75616.3 | valid tokens/s: 75616.3 | physical tokens/s: 76256.8 | loss tokens/s: 58473.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.977738E-01 | lm-global-supervised-tokens: 1.297369E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.869935E-02 | lm-token-buffer-mib: 2.352827E+02 | global_load_balancing_loss: 1.001130E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11157.864 | params rms: 0.060416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:38:12.458994 | finish at 2026-10-17 19:02:02 + [2026-10-16 08:27:33] iteration 1823/ 2384 | consumed samples: 58336 | elapsed time per iteration (ms): 222528.1 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 75393.6 | valid tokens/s: 75393.6 | physical tokens/s: 76258.5 | loss tokens/s: 62765.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.788747E-01 | lm-global-supervised-tokens: 1.396707E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.885098E-02 | lm-token-buffer-mib: 2.349898E+02 | global_load_balancing_loss: 1.000523E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11157.833 | params rms: 0.060416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:40:38.260496 | finish at 2026-10-17 19:08:11 + [2026-10-16 08:31:16] iteration 1824/ 2384 | consumed samples: 58368 | elapsed time per iteration (ms): 223640.6 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 75018.6 | valid tokens/s: 75018.6 | physical tokens/s: 75553.8 | loss tokens/s: 54859.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.187051E-01 | lm-global-supervised-tokens: 1.226885E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.353452E-02 | lm-token-buffer-mib: 2.334401E+02 | global_load_balancing_loss: 1.003178E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11157.804 | params rms: 0.060416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:47:18.726368 | finish at 2026-10-17 19:18:35 + [2026-10-16 08:34:59] iteration 1825/ 2384 | consumed samples: 58400 | elapsed time per iteration (ms): 222417.6 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 75431.1 | valid tokens/s: 75431.1 | physical tokens/s: 76053.0 | loss tokens/s: 55447.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.143096E-01 | lm-global-supervised-tokens: 1.233239E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.959020E-02 | lm-token-buffer-mib: 2.335726E+02 | global_load_balancing_loss: 1.001437E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11157.773 | params rms: 0.060416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:32:11.456346 | finish at 2026-10-17 19:07:10 + [2026-10-16 08:38:42] iteration 1826/ 2384 | consumed samples: 58432 | elapsed time per iteration (ms): 223725.9 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 74989.9 | valid tokens/s: 74989.9 | physical tokens/s: 76134.9 | loss tokens/s: 56676.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.644354E-01 | lm-global-supervised-tokens: 1.267992E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.432907E-02 | lm-token-buffer-mib: 2.356425E+02 | global_load_balancing_loss: 1.000155E+00 | loss scale: 1.0 | grad norm: 0.508 | params norm: 11157.744 | params rms: 0.060416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:40:39.027552 | finish at 2026-10-17 19:19:21 + [2026-10-16 08:42:26] iteration 1827/ 2384 | consumed samples: 58464 | elapsed time per iteration (ms): 223184.8 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 75171.8 | valid tokens/s: 75171.8 | physical tokens/s: 75709.2 | loss tokens/s: 52662.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.018597E-01 | lm-global-supervised-tokens: 1.175356E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.084621E-02 | lm-token-buffer-mib: 2.326871E+02 | global_load_balancing_loss: 1.000460E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11157.715 | params rms: 0.060415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:31:53.926777 | finish at 2026-10-17 19:14:19 + [2026-10-16 08:46:06] iteration 1828/ 2384 | consumed samples: 58496 | elapsed time per iteration (ms): 220970.9 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 75924.8 | valid tokens/s: 75924.8 | physical tokens/s: 76570.0 | loss tokens/s: 54488.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.499922E-01 | lm-global-supervised-tokens: 1.204030E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.986013E-02 | lm-token-buffer-mib: 2.339456E+02 | global_load_balancing_loss: 9.994376E-01 | loss scale: 1.0 | grad norm: 0.375 | params norm: 11157.684 | params rms: 0.060415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:07:39.823216 | finish at 2026-10-17 18:53:46 + [2026-10-16 08:49:50] iteration 1829/ 2384 | consumed samples: 58528 | elapsed time per iteration (ms): 223058.6 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75214.4 | valid tokens/s: 75214.3 | physical tokens/s: 75704.2 | loss tokens/s: 52720.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.132885E-01 | lm-global-supervised-tokens: 1.175970E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.889731E-02 | lm-token-buffer-mib: 2.342433E+02 | global_load_balancing_loss: 9.984314E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11157.653 | params rms: 0.060415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:23:17.497036 | finish at 2026-10-17 19:13:07 + [2026-10-16 08:53:57] iteration 1830/ 2384 | consumed samples: 58560 | elapsed time per iteration (ms): 247359.3 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 67825.2 | valid tokens/s: 67825.2 | physical tokens/s: 68345.6 | loss tokens/s: 51019.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.227042E-01 | lm-global-supervised-tokens: 1.262020E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.149157E-02 | lm-token-buffer-mib: 2.343871E+02 | global_load_balancing_loss: 9.990314E-01 | loss scale: 1.0 | grad norm: 0.243 | params norm: 11157.623 | params rms: 0.060415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:03:57.056502 | finish at 2026-10-17 22:57:54 + [2026-10-16 08:57:38] iteration 1831/ 2384 | consumed samples: 58592 | elapsed time per iteration (ms): 221512.4 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.24% | FLOPs estimator: workload-aware | logical tokens/s: 75739.2 | valid tokens/s: 75739.2 | physical tokens/s: 76657.6 | loss tokens/s: 60525.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.712974E-01 | lm-global-supervised-tokens: 1.340703E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.354048E-02 | lm-token-buffer-mib: 2.335373E+02 | global_load_balancing_loss: 1.000979E+00 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11157.591 | params rms: 0.060415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:01:36.371654 | finish at 2026-10-17 18:59:15 + [2026-10-16 09:01:19] iteration 1832/ 2384 | consumed samples: 58624 | elapsed time per iteration (ms): 220469.7 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 76097.5 | valid tokens/s: 76097.5 | physical tokens/s: 76428.8 | loss tokens/s: 54803.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.888142E-01 | lm-global-supervised-tokens: 1.208256E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.637781E-02 | lm-token-buffer-mib: 2.324161E+02 | global_load_balancing_loss: 1.001651E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11157.557 | params rms: 0.060415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:48:19.252607 | finish at 2026-10-17 18:49:38 + [2026-10-16 09:05:04] iteration 1833/ 2384 | consumed samples: 58656 | elapsed time per iteration (ms): 224723.2 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 74657.2 | valid tokens/s: 74657.2 | physical tokens/s: 75741.8 | loss tokens/s: 57910.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.100664E-01 | lm-global-supervised-tokens: 1.301390E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.873377E-02 | lm-token-buffer-mib: 2.344669E+02 | global_load_balancing_loss: 1.001756E+00 | loss scale: 1.0 | grad norm: 0.183 | params norm: 11157.524 | params rms: 0.060414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:23:42.463146 | finish at 2026-10-17 19:28:46 + [2026-10-16 09:08:46] iteration 1834/ 2384 | consumed samples: 58688 | elapsed time per iteration (ms): 222561.4 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 75382.4 | valid tokens/s: 75382.4 | physical tokens/s: 76255.5 | loss tokens/s: 59496.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.670310E-01 | lm-global-supervised-tokens: 1.324167E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.187831E-02 | lm-token-buffer-mib: 2.359622E+02 | global_load_balancing_loss: 1.001379E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11157.491 | params rms: 0.060414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:00:08.750820 | finish at 2026-10-17 19:08:55 + [2026-10-16 09:12:31] iteration 1835/ 2384 | consumed samples: 58720 | elapsed time per iteration (ms): 224436.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 74752.6 | valid tokens/s: 74752.6 | physical tokens/s: 75602.0 | loss tokens/s: 59178.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.192533E-01 | lm-global-supervised-tokens: 1.328180E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.702061E-02 | lm-token-buffer-mib: 2.342811E+02 | global_load_balancing_loss: 1.000435E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11157.457 | params rms: 0.060414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:13:35.423746 | finish at 2026-10-17 19:26:06 + [2026-10-16 09:16:10] iteration 1836/ 2384 | consumed samples: 58752 | elapsed time per iteration (ms): 219868.6 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 76305.6 | valid tokens/s: 76305.6 | physical tokens/s: 77038.6 | loss tokens/s: 59972.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.602989E-01 | lm-global-supervised-tokens: 1.318608E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.387275E-02 | lm-token-buffer-mib: 2.331925E+02 | global_load_balancing_loss: 9.998122E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11157.421 | params rms: 0.060414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:28:07.980590 | finish at 2026-10-17 18:44:18 + [2026-10-16 09:19:54] iteration 1837/ 2384 | consumed samples: 58784 | elapsed time per iteration (ms): 223649.2 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 75015.6 | valid tokens/s: 75015.6 | physical tokens/s: 75551.2 | loss tokens/s: 55736.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.229550E-01 | lm-global-supervised-tokens: 1.246544E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.174759E-02 | lm-token-buffer-mib: 2.332361E+02 | global_load_balancing_loss: 9.994258E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11157.388 | params rms: 0.060414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:58:56.115249 | finish at 2026-10-17 19:18:50 + [2026-10-16 09:23:37] iteration 1838/ 2384 | consumed samples: 58816 | elapsed time per iteration (ms): 223338.3 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.32% | FLOPs estimator: workload-aware | logical tokens/s: 75120.0 | valid tokens/s: 75120.0 | physical tokens/s: 75855.5 | loss tokens/s: 55677.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.302218E-01 | lm-global-supervised-tokens: 1.243489E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.871239E-02 | lm-token-buffer-mib: 2.342387E+02 | global_load_balancing_loss: 9.998024E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11157.359 | params rms: 0.060413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:52:22.713096 | finish at 2026-10-17 19:16:00 + [2026-10-16 09:27:22] iteration 1839/ 2384 | consumed samples: 58848 | elapsed time per iteration (ms): 224714.0 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 74660.0 | valid tokens/s: 74660.0 | physical tokens/s: 75493.6 | loss tokens/s: 58666.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.324567E-01 | lm-global-supervised-tokens: 1.318311E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.678088E-02 | lm-token-buffer-mib: 2.358339E+02 | global_load_balancing_loss: 9.997333E-01 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11157.329 | params rms: 0.060413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:01:09.149613 | finish at 2026-10-17 19:28:31 + [2026-10-16 09:31:28] iteration 1840/ 2384 | consumed samples: 58880 | elapsed time per iteration (ms): 245652.8 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 68296.3 | valid tokens/s: 68296.3 | physical tokens/s: 68986.4 | loss tokens/s: 52319.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.443852E-01 | lm-global-supervised-tokens: 1.285237E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.148386E-02 | lm-token-buffer-mib: 2.350104E+02 | global_load_balancing_loss: 9.991933E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11157.297 | params rms: 0.060413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:07:15.102364 | finish at 2026-10-17 22:38:43 +times across ranks (ms): + save-checkpoint: + rank 0: 25029.96 + rank 1: 25029.95 + rank 2: 25030.04 + rank 3: 25029.97 + rank 4: 25029.96 + rank 5: 25029.99 + rank 6: 25029.97 + rank 7: 25029.98 + rank 8: 25029.69 + rank 9: 25029.65 + rank 10: 25029.70 + rank 11: 25029.66 + rank 12: 25029.67 + rank 13: 25029.77 + rank 14: 25029.67 + rank 15: 25029.69 + rank 16: 25030.30 + rank 17: 25030.29 + rank 18: 25030.34 + rank 19: 25030.37 + rank 20: 25030.25 + rank 21: 25030.26 + rank 22: 25030.29 + rank 23: 25030.25 + rank 24: 25029.73 + rank 25: 25029.73 + rank 26: 25029.79 + rank 27: 25029.71 + rank 28: 25029.72 + rank 29: 25029.76 + rank 30: 25029.69 + rank 31: 25029.98 + rank 32: 25030.12 + rank 33: 25030.14 + rank 34: 25030.12 + rank 35: 25030.15 + rank 36: 25030.17 + rank 37: 25030.12 + rank 38: 25030.08 + rank 39: 25030.15 + rank 40: 25029.71 + rank 41: 25029.69 + rank 42: 25029.68 + rank 43: 25029.69 + rank 44: 25029.66 + rank 45: 25029.70 + rank 46: 25029.71 + rank 47: 25029.65 + rank 48: 25029.85 + rank 49: 25029.87 + rank 50: 25029.89 + rank 51: 25029.82 + rank 52: 25029.83 + rank 53: 25029.84 + rank 54: 25029.85 + rank 55: 25029.81 + rank 56: 25030.50 + rank 57: 25030.48 + rank 58: 25030.51 + rank 59: 25030.38 + rank 60: 25030.57 + rank 61: 25030.47 + rank 62: 25030.23 + rank 63: 25030.53 + [2026-10-16 09:35:41] iteration 1841/ 2384 | consumed samples: 58912 | elapsed time per iteration (ms): 228369.9 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 73465.0 | valid tokens/s: 73465.0 | physical tokens/s: 74166.0 | loss tokens/s: 54856.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.648835E-01 | lm-global-supervised-tokens: 1.252763E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.534974E-02 | lm-token-buffer-mib: 2.344732E+02 | global_load_balancing_loss: 1.000474E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11157.265 | params rms: 0.060413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:26:44.852328 | finish at 2026-10-17 20:02:26 + [2026-10-16 09:39:29] iteration 1842/ 2384 | consumed samples: 58944 | elapsed time per iteration (ms): 227404.1 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 73777.0 | valid tokens/s: 73777.0 | physical tokens/s: 74407.5 | loss tokens/s: 58641.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.052635E-01 | lm-global-supervised-tokens: 1.333532E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.392507E-02 | lm-token-buffer-mib: 2.335406E+02 | global_load_balancing_loss: 1.001464E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11157.231 | params rms: 0.060413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:14:13.045041 | finish at 2026-10-17 19:53:42 + [2026-10-16 09:43:14] iteration 1843/ 2384 | consumed samples: 58976 | elapsed time per iteration (ms): 224887.6 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 74602.4 | valid tokens/s: 74602.4 | physical tokens/s: 75541.8 | loss tokens/s: 54056.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.305421E-01 | lm-global-supervised-tokens: 1.215655E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.545964E-02 | lm-token-buffer-mib: 2.361125E+02 | global_load_balancing_loss: 1.001780E+00 | loss scale: 1.0 | grad norm: 0.307 | params norm: 11157.196 | params rms: 0.060413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:47:44.217496 | finish at 2026-10-17 19:30:58 + [2026-10-16 09:46:55] iteration 1844/ 2384 | consumed samples: 59008 | elapsed time per iteration (ms): 221589.3 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.79% | FLOPs estimator: workload-aware | logical tokens/s: 75712.9 | valid tokens/s: 75712.9 | physical tokens/s: 76535.1 | loss tokens/s: 55957.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.729006E-01 | lm-global-supervised-tokens: 1.239953E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.994652E-02 | lm-token-buffer-mib: 2.353939E+02 | global_load_balancing_loss: 1.000229E+00 | loss scale: 1.0 | grad norm: 0.236 | params norm: 11157.162 | params rms: 0.060412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:14:18.240623 | finish at 2026-10-17 19:01:14 + [2026-10-16 09:50:34] iteration 1845/ 2384 | consumed samples: 59040 | elapsed time per iteration (ms): 218906.4 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.06% | FLOPs estimator: workload-aware | logical tokens/s: 76640.9 | valid tokens/s: 76640.9 | physical tokens/s: 77122.3 | loss tokens/s: 54771.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.906090E-01 | lm-global-supervised-tokens: 1.198972E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.704438E-02 | lm-token-buffer-mib: 2.337941E+02 | global_load_balancing_loss: 1.000973E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11157.127 | params rms: 0.060412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:46:30.523366 | finish at 2026-10-17 18:37:05 + [2026-10-16 09:54:18] iteration 1846/ 2384 | consumed samples: 59072 | elapsed time per iteration (ms): 223611.5 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 75028.4 | valid tokens/s: 75028.4 | physical tokens/s: 75827.2 | loss tokens/s: 55164.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.551230E-01 | lm-global-supervised-tokens: 1.233542E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.424349E-02 | lm-token-buffer-mib: 2.338699E+02 | global_load_balancing_loss: 1.000145E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11157.095 | params rms: 0.060412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:25:02.963540 | finish at 2026-10-17 19:19:21 + [2026-10-16 09:57:59] iteration 1847/ 2384 | consumed samples: 59104 | elapsed time per iteration (ms): 221454.3 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75759.1 | valid tokens/s: 75759.1 | physical tokens/s: 76495.6 | loss tokens/s: 55978.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.038835E-01 | lm-global-supervised-tokens: 1.239660E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.803540E-02 | lm-token-buffer-mib: 2.345125E+02 | global_load_balancing_loss: 9.994586E-01 | loss scale: 1.0 | grad norm: 0.233 | params norm: 11157.059 | params rms: 0.060412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:02:00.948434 | finish at 2026-10-17 19:00:00 + [2026-10-16 10:01:40] iteration 1848/ 2384 | consumed samples: 59136 | elapsed time per iteration (ms): 220869.7 | throughput per GPU (TFLOP/s/GPU): 26.1 | MFU 8.35% | FLOPs estimator: workload-aware | logical tokens/s: 75959.7 | valid tokens/s: 75959.7 | physical tokens/s: 76355.6 | loss tokens/s: 56023.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.850871E-01 | lm-global-supervised-tokens: 1.237388E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.291261E-02 | lm-token-buffer-mib: 2.331988E+02 | global_load_balancing_loss: 9.999731E-01 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11157.023 | params rms: 0.060412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:53:06.177067 | finish at 2026-10-17 18:54:46 + [2026-10-16 10:05:21] iteration 1849/ 2384 | consumed samples: 59168 | elapsed time per iteration (ms): 221239.9 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 75832.6 | valid tokens/s: 75832.6 | physical tokens/s: 76443.9 | loss tokens/s: 58094.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.211293E-01 | lm-global-supervised-tokens: 1.285276E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.310125E-02 | lm-token-buffer-mib: 2.330860E+02 | global_load_balancing_loss: 9.992919E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11156.985 | params rms: 0.060411 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:52:43.346688 | finish at 2026-10-17 18:58:05 + [2026-10-16 10:09:24] iteration 1850/ 2384 | consumed samples: 59200 | elapsed time per iteration (ms): 242814.6 | throughput per GPU (TFLOP/s/GPU): 18.4 | MFU 5.91% | FLOPs estimator: workload-aware | logical tokens/s: 69094.7 | valid tokens/s: 69094.7 | physical tokens/s: 69741.0 | loss tokens/s: 56227.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.744374E-01 | lm-global-supervised-tokens: 1.365277E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.091119E-02 | lm-token-buffer-mib: 2.342262E+02 | global_load_balancing_loss: 1.000279E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11156.947 | params rms: 0.060411 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:01:02.987738 | finish at 2026-10-17 22:10:27 + [2026-10-16 10:13:11] iteration 1851/ 2384 | consumed samples: 59232 | elapsed time per iteration (ms): 226369.3 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 74114.3 | valid tokens/s: 74114.3 | physical tokens/s: 75483.9 | loss tokens/s: 54830.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.371049E-01 | lm-global-supervised-tokens: 1.241200E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.412222E-02 | lm-token-buffer-mib: 2.385817E+02 | global_load_balancing_loss: 1.001006E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11156.911 | params rms: 0.060411 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:30:54.854250 | finish at 2026-10-17 19:44:06 + [2026-10-16 10:16:54] iteration 1852/ 2384 | consumed samples: 59264 | elapsed time per iteration (ms): 223227.8 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 75157.4 | valid tokens/s: 75157.4 | physical tokens/s: 75637.6 | loss tokens/s: 60829.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.907415E-01 | lm-global-supervised-tokens: 1.357887E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.252840E-02 | lm-token-buffer-mib: 2.332849E+02 | global_load_balancing_loss: 1.000803E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11156.874 | params rms: 0.060411 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:59:17.175083 | finish at 2026-10-17 19:16:11 + [2026-10-16 10:20:35] iteration 1853/ 2384 | consumed samples: 59296 | elapsed time per iteration (ms): 221554.5 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 75724.9 | valid tokens/s: 75724.9 | physical tokens/s: 76845.2 | loss tokens/s: 59123.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.611331E-01 | lm-global-supervised-tokens: 1.309917E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.521993E-02 | lm-token-buffer-mib: 2.345241E+02 | global_load_balancing_loss: 1.000540E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11156.840 | params rms: 0.060411 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:40:45.465381 | finish at 2026-10-17 19:01:21 + [2026-10-16 10:24:16] iteration 1854/ 2384 | consumed samples: 59328 | elapsed time per iteration (ms): 220908.7 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75946.3 | valid tokens/s: 75946.3 | physical tokens/s: 76779.3 | loss tokens/s: 60972.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.444265E-01 | lm-global-supervised-tokens: 1.346924E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.656771E-02 | lm-token-buffer-mib: 2.334124E+02 | global_load_balancing_loss: 1.001020E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11156.804 | params rms: 0.060410 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:31:21.605182 | finish at 2026-10-17 18:55:38 + [2026-10-16 10:27:58] iteration 1855/ 2384 | consumed samples: 59360 | elapsed time per iteration (ms): 221508.9 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 75740.5 | valid tokens/s: 75740.5 | physical tokens/s: 76577.9 | loss tokens/s: 58621.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.943190E-01 | lm-global-supervised-tokens: 1.298521E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.593557E-02 | lm-token-buffer-mib: 2.342348E+02 | global_load_balancing_loss: 1.000116E+00 | loss scale: 1.0 | grad norm: 0.291 | params norm: 11156.765 | params rms: 0.060410 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:32:58.199359 | finish at 2026-10-17 19:00:56 + [2026-10-16 10:31:43] iteration 1856/ 2384 | consumed samples: 59392 | elapsed time per iteration (ms): 224949.0 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 74582.1 | valid tokens/s: 74582.1 | physical tokens/s: 75056.3 | loss tokens/s: 60171.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.217186E-01 | lm-global-supervised-tokens: 1.353558E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.173219E-02 | lm-token-buffer-mib: 2.329725E+02 | global_load_balancing_loss: 1.000411E+00 | loss scale: 1.0 | grad norm: 0.294 | params norm: 11156.727 | params rms: 0.060410 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:59:33.084148 | finish at 2026-10-17 19:31:16 + [2026-10-16 10:35:24] iteration 1857/ 2384 | consumed samples: 59424 | elapsed time per iteration (ms): 221694.0 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75677.3 | valid tokens/s: 75677.3 | physical tokens/s: 76570.6 | loss tokens/s: 59925.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.788886E-01 | lm-global-supervised-tokens: 1.328505E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.092464E-02 | lm-token-buffer-mib: 2.333222E+02 | global_load_balancing_loss: 9.990840E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11156.692 | params rms: 0.060410 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:27:12.718151 | finish at 2026-10-17 19:02:37 + [2026-10-16 10:39:09] iteration 1858/ 2384 | consumed samples: 59456 | elapsed time per iteration (ms): 224316.2 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.88% | FLOPs estimator: workload-aware | logical tokens/s: 74792.5 | valid tokens/s: 74792.5 | physical tokens/s: 75622.7 | loss tokens/s: 54569.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.484378E-01 | lm-global-supervised-tokens: 1.224074E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.252181E-02 | lm-token-buffer-mib: 2.336405E+02 | global_load_balancing_loss: 9.999619E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11156.654 | params rms: 0.060410 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:46:30.325975 | finish at 2026-10-17 19:25:39 + [2026-10-16 10:42:49] iteration 1859/ 2384 | consumed samples: 59488 | elapsed time per iteration (ms): 219973.4 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 76269.2 | valid tokens/s: 76269.2 | physical tokens/s: 76942.2 | loss tokens/s: 57896.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.927900E-01 | lm-global-supervised-tokens: 1.273570E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.233616E-02 | lm-token-buffer-mib: 2.339248E+02 | global_load_balancing_loss: 9.996926E-01 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11156.617 | params rms: 0.060409 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:04:46.044449 | finish at 2026-10-17 18:47:35 + [2026-10-16 10:46:52] iteration 1860/ 2384 | consumed samples: 59520 | elapsed time per iteration (ms): 243347.1 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.66% | FLOPs estimator: workload-aware | logical tokens/s: 68943.4 | valid tokens/s: 68943.4 | physical tokens/s: 69506.1 | loss tokens/s: 51263.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.128107E-01 | lm-global-supervised-tokens: 1.247490E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.609226E-02 | lm-token-buffer-mib: 2.343008E+02 | global_load_balancing_loss: 1.001627E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11156.580 | params rms: 0.060409 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:25:13.860296 | finish at 2026-10-17 22:12:06 + [2026-10-16 10:50:37] iteration 1861/ 2384 | consumed samples: 59552 | elapsed time per iteration (ms): 225271.5 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 74475.3 | valid tokens/s: 74475.3 | physical tokens/s: 75677.6 | loss tokens/s: 58470.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.652899E-01 | lm-global-supervised-tokens: 1.317176E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.598177E-02 | lm-token-buffer-mib: 2.363957E+02 | global_load_balancing_loss: 1.001413E+00 | loss scale: 1.0 | grad norm: 0.272 | params norm: 11156.542 | params rms: 0.060409 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:43:36.974981 | finish at 2026-10-17 19:34:14 + [2026-10-16 10:54:18] iteration 1862/ 2384 | consumed samples: 59584 | elapsed time per iteration (ms): 220824.2 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75975.2 | valid tokens/s: 75975.2 | physical tokens/s: 76403.8 | loss tokens/s: 56622.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.831258E-01 | lm-global-supervised-tokens: 1.250352E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.293537E-02 | lm-token-buffer-mib: 2.337512E+02 | global_load_balancing_loss: 1.001103E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11156.506 | params rms: 0.060409 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:01:10.249779 | finish at 2026-10-17 18:55:28 + [2026-10-16 10:58:01] iteration 1863/ 2384 | consumed samples: 59616 | elapsed time per iteration (ms): 222913.9 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 75263.1 | valid tokens/s: 75263.1 | physical tokens/s: 75797.8 | loss tokens/s: 51405.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.060967E-01 | lm-global-supervised-tokens: 1.145908E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.503069E-02 | lm-token-buffer-mib: 2.341762E+02 | global_load_balancing_loss: 1.001025E+00 | loss scale: 1.0 | grad norm: 0.258 | params norm: 11156.471 | params rms: 0.060409 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:15:38.164703 | finish at 2026-10-17 19:13:39 + [2026-10-16 11:01:42] iteration 1864/ 2384 | consumed samples: 59648 | elapsed time per iteration (ms): 221349.1 | throughput per GPU (TFLOP/s/GPU): 18.3 | MFU 5.88% | FLOPs estimator: workload-aware | logical tokens/s: 75795.2 | valid tokens/s: 75795.2 | physical tokens/s: 76801.2 | loss tokens/s: 57594.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.741635E-01 | lm-global-supervised-tokens: 1.274855E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.172698E-02 | lm-token-buffer-mib: 2.336312E+02 | global_load_balancing_loss: 9.995048E-01 | loss scale: 1.0 | grad norm: 0.301 | params norm: 11156.438 | params rms: 0.060408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:58:21.509743 | finish at 2026-10-17 19:00:04 + [2026-10-16 11:05:24] iteration 1865/ 2384 | consumed samples: 59680 | elapsed time per iteration (ms): 221355.5 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 75793.0 | valid tokens/s: 75793.0 | physical tokens/s: 77201.1 | loss tokens/s: 58039.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.889143E-01 | lm-global-supervised-tokens: 1.284739E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.322927E-02 | lm-token-buffer-mib: 2.340764E+02 | global_load_balancing_loss: 9.992508E-01 | loss scale: 1.0 | grad norm: 0.230 | params norm: 11156.406 | params rms: 0.060408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:54:43.508822 | finish at 2026-10-17 19:00:07 + [2026-10-16 11:09:10] iteration 1866/ 2384 | consumed samples: 59712 | elapsed time per iteration (ms): 225747.9 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 74318.3 | valid tokens/s: 74318.3 | physical tokens/s: 75076.7 | loss tokens/s: 54707.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.307037E-01 | lm-global-supervised-tokens: 1.235007E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.955289E-02 | lm-token-buffer-mib: 2.352199E+02 | global_load_balancing_loss: 9.993988E-01 | loss scale: 1.0 | grad norm: 0.265 | params norm: 11156.375 | params rms: 0.060408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:28:57.400225 | finish at 2026-10-17 19:38:07 + [2026-10-16 11:12:51] iteration 1867/ 2384 | consumed samples: 59744 | elapsed time per iteration (ms): 221674.1 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 75684.0 | valid tokens/s: 75683.9 | physical tokens/s: 76412.9 | loss tokens/s: 57156.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.555119E-01 | lm-global-supervised-tokens: 1.267001E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.068924E-02 | lm-token-buffer-mib: 2.349969E+02 | global_load_balancing_loss: 1.000204E+00 | loss scale: 1.0 | grad norm: 0.196 | params norm: 11156.345 | params rms: 0.060408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:50:05.522233 | finish at 2026-10-17 19:02:57 + [2026-10-16 11:16:36] iteration 1868/ 2384 | consumed samples: 59776 | elapsed time per iteration (ms): 225076.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 74539.8 | valid tokens/s: 74539.8 | physical tokens/s: 75832.4 | loss tokens/s: 61958.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.756235E-01 | lm-global-supervised-tokens: 1.394549E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.427016E-02 | lm-token-buffer-mib: 2.379537E+02 | global_load_balancing_loss: 1.000891E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11156.315 | params rms: 0.060408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:15:39.482826 | finish at 2026-10-17 19:32:16 + [2026-10-16 11:20:18] iteration 1869/ 2384 | consumed samples: 59808 | elapsed time per iteration (ms): 221605.6 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.47% | FLOPs estimator: workload-aware | logical tokens/s: 75707.5 | valid tokens/s: 75707.5 | physical tokens/s: 76363.4 | loss tokens/s: 55314.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.199038E-01 | lm-global-supervised-tokens: 1.225799E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.233722E-02 | lm-token-buffer-mib: 2.337784E+02 | global_load_balancing_loss: 1.000188E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11156.282 | params rms: 0.060408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:42:06.880009 | finish at 2026-10-17 19:02:25 + [2026-10-16 11:24:26] iteration 1870/ 2384 | consumed samples: 59840 | elapsed time per iteration (ms): 248128.1 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.09% | FLOPs estimator: workload-aware | logical tokens/s: 67615.0 | valid tokens/s: 67615.0 | physical tokens/s: 68857.3 | loss tokens/s: 53847.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.359414E-01 | lm-global-supervised-tokens: 1.336116E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.823666E-02 | lm-token-buffer-mib: 2.357551E+02 | global_load_balancing_loss: 1.001793E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11156.252 | params rms: 0.060407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:25:37.837721 | finish at 2026-10-17 22:50:04 + [2026-10-16 11:28:11] iteration 1871/ 2384 | consumed samples: 59872 | elapsed time per iteration (ms): 224825.5 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.07% | FLOPs estimator: workload-aware | logical tokens/s: 74623.1 | valid tokens/s: 74623.1 | physical tokens/s: 74789.8 | loss tokens/s: 56367.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.877275E-01 | lm-global-supervised-tokens: 1.267286E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.507755E-02 | lm-token-buffer-mib: 2.325131E+02 | global_load_balancing_loss: 1.000761E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11156.221 | params rms: 0.060407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:02:15.497284 | finish at 2026-10-17 19:30:26 + [2026-10-16 11:31:52] iteration 1872/ 2384 | consumed samples: 59904 | elapsed time per iteration (ms): 221407.2 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.49% | FLOPs estimator: workload-aware | logical tokens/s: 75775.2 | valid tokens/s: 75775.2 | physical tokens/s: 76103.7 | loss tokens/s: 52896.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.468046E-01 | lm-global-supervised-tokens: 1.171167E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.407037E-02 | lm-token-buffer-mib: 2.326440E+02 | global_load_balancing_loss: 1.000135E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11156.190 | params rms: 0.060407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:29:20.503296 | finish at 2026-10-17 19:01:13 + [2026-10-16 11:35:32] iteration 1873/ 2384 | consumed samples: 59936 | elapsed time per iteration (ms): 219603.7 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 76397.6 | valid tokens/s: 76397.6 | physical tokens/s: 76999.8 | loss tokens/s: 53111.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.782350E-01 | lm-global-supervised-tokens: 1.166337E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.542556E-02 | lm-token-buffer-mib: 2.342599E+02 | global_load_balancing_loss: 1.001066E+00 | loss scale: 1.0 | grad norm: 0.214 | params norm: 11156.159 | params rms: 0.060407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:10:17.486396 | finish at 2026-10-17 18:45:49 + [2026-10-16 11:39:12] iteration 1874/ 2384 | consumed samples: 59968 | elapsed time per iteration (ms): 220296.7 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 76157.3 | valid tokens/s: 76157.3 | physical tokens/s: 76545.2 | loss tokens/s: 58406.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.236513E-01 | lm-global-supervised-tokens: 1.286675E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.323757E-02 | lm-token-buffer-mib: 2.335601E+02 | global_load_balancing_loss: 1.000605E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11156.124 | params rms: 0.060407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:12:31.329038 | finish at 2026-10-17 18:51:44 + [2026-10-16 11:42:57] iteration 1875/ 2384 | consumed samples: 60000 | elapsed time per iteration (ms): 224908.1 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 74595.8 | valid tokens/s: 74595.8 | physical tokens/s: 75254.6 | loss tokens/s: 58557.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.438836E-01 | lm-global-supervised-tokens: 1.317000E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.680231E-02 | lm-token-buffer-mib: 2.346082E+02 | global_load_balancing_loss: 1.000024E+00 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11156.091 | params rms: 0.060407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:47:58.226242 | finish at 2026-10-17 19:30:55 + [2026-10-16 11:46:39] iteration 1876/ 2384 | consumed samples: 60032 | elapsed time per iteration (ms): 221475.5 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 75752.0 | valid tokens/s: 75752.0 | physical tokens/s: 76360.6 | loss tokens/s: 52629.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.292490E-01 | lm-global-supervised-tokens: 1.165611E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.584062E-02 | lm-token-buffer-mib: 2.342663E+02 | global_load_balancing_loss: 9.998960E-01 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11156.060 | params rms: 0.060406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:15:09.547877 | finish at 2026-10-17 19:01:48 + [2026-10-16 11:50:25] iteration 1877/ 2384 | consumed samples: 60064 | elapsed time per iteration (ms): 226451.6 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 74087.4 | valid tokens/s: 74087.4 | physical tokens/s: 75063.3 | loss tokens/s: 54692.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.145229E-01 | lm-global-supervised-tokens: 1.238518E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.514899E-02 | lm-token-buffer-mib: 2.367019E+02 | global_load_balancing_loss: 9.992715E-01 | loss scale: 1.0 | grad norm: 0.276 | params norm: 11156.031 | params rms: 0.060406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:53:30.942623 | finish at 2026-10-17 19:43:56 + [2026-10-16 11:54:08] iteration 1878/ 2384 | consumed samples: 60096 | elapsed time per iteration (ms): 223066.3 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75211.7 | valid tokens/s: 75211.7 | physical tokens/s: 75777.9 | loss tokens/s: 55722.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.954510E-01 | lm-global-supervised-tokens: 1.242985E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.529195E-02 | lm-token-buffer-mib: 2.334133E+02 | global_load_balancing_loss: 9.993212E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11156.005 | params rms: 0.060406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:21:11.552583 | finish at 2026-10-17 19:15:20 + [2026-10-16 11:57:54] iteration 1879/ 2384 | consumed samples: 60128 | elapsed time per iteration (ms): 225471.1 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.44% | FLOPs estimator: workload-aware | logical tokens/s: 74409.5 | valid tokens/s: 74409.5 | physical tokens/s: 75351.7 | loss tokens/s: 54404.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.463802E-01 | lm-global-supervised-tokens: 1.226659E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.393156E-02 | lm-token-buffer-mib: 2.367502E+02 | global_load_balancing_loss: 1.000413E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11155.974 | params rms: 0.060406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:37:42.903018 | finish at 2026-10-17 19:35:36 + [2026-10-16 12:01:58] iteration 1880/ 2384 | consumed samples: 60160 | elapsed time per iteration (ms): 244677.5 | throughput per GPU (TFLOP/s/GPU): 18.3 | MFU 5.87% | FLOPs estimator: workload-aware | logical tokens/s: 68568.7 | valid tokens/s: 68568.7 | physical tokens/s: 69449.1 | loss tokens/s: 52418.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.944299E-01 | lm-global-supervised-tokens: 1.282560E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.818819E-02 | lm-token-buffer-mib: 2.362838E+02 | global_load_balancing_loss: 1.001350E+00 | loss scale: 1.0 | grad norm: 0.244 | params norm: 11155.943 | params rms: 0.060406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:15:17.471781 | finish at 2026-10-17 22:17:16 +times across ranks (ms): + save-checkpoint: + rank 0: 25610.91 + rank 1: 25610.89 + rank 2: 25610.90 + rank 3: 25610.90 + rank 4: 25610.88 + rank 5: 25610.96 + rank 6: 25610.90 + rank 7: 25610.73 + rank 8: 25611.30 + rank 9: 25611.32 + rank 10: 25611.33 + rank 11: 25611.29 + rank 12: 25611.29 + rank 13: 25611.30 + rank 14: 25611.29 + rank 15: 25611.30 + rank 16: 25611.10 + rank 17: 25611.12 + rank 18: 25611.08 + rank 19: 25611.13 + rank 20: 25611.07 + rank 21: 25611.07 + rank 22: 25611.14 + rank 23: 25611.06 + rank 24: 25611.48 + rank 25: 25611.47 + rank 26: 25611.54 + rank 27: 25611.61 + rank 28: 25611.47 + rank 29: 25611.49 + rank 30: 25611.48 + rank 31: 25611.52 + rank 32: 25611.17 + rank 33: 25611.16 + rank 34: 25611.28 + rank 35: 25611.14 + rank 36: 25611.12 + rank 37: 25611.27 + rank 38: 25611.12 + rank 39: 25611.12 + rank 40: 25611.50 + rank 41: 25611.49 + rank 42: 25611.50 + rank 43: 25611.50 + rank 44: 25611.50 + rank 45: 25611.51 + rank 46: 25611.46 + rank 47: 25611.48 + rank 48: 25611.57 + rank 49: 25611.60 + rank 50: 25611.67 + rank 51: 25611.56 + rank 52: 25611.71 + rank 53: 25611.67 + rank 54: 25611.65 + rank 55: 25611.55 + rank 56: 25611.29 + rank 57: 25611.34 + rank 58: 25611.33 + rank 59: 25611.34 + rank 60: 25611.33 + rank 61: 25611.29 + rank 62: 25611.32 + rank 63: 25611.28 + [2026-10-16 12:06:22] iteration 1881/ 2384 | consumed samples: 60192 | elapsed time per iteration (ms): 237773.4 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.20% | FLOPs estimator: workload-aware | logical tokens/s: 70559.6 | valid tokens/s: 70559.5 | physical tokens/s: 71085.6 | loss tokens/s: 53268.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.552338E-01 | lm-global-supervised-tokens: 1.266591E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.432596E-02 | lm-token-buffer-mib: 2.343615E+02 | global_load_balancing_loss: 1.001514E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11155.913 | params rms: 0.060406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:13:20.039662 | finish at 2026-10-17 21:19:42 + [2026-10-16 12:10:10] iteration 1882/ 2384 | consumed samples: 60224 | elapsed time per iteration (ms): 228359.1 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.83% | FLOPs estimator: workload-aware | logical tokens/s: 73468.3 | valid tokens/s: 73468.3 | physical tokens/s: 74156.1 | loss tokens/s: 52111.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.651234E-01 | lm-global-supervised-tokens: 1.190020E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.540768E-02 | lm-token-buffer-mib: 2.339299E+02 | global_load_balancing_loss: 1.001334E+00 | loss scale: 1.0 | grad norm: 0.248 | params norm: 11155.882 | params rms: 0.060405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:50:36.246589 | finish at 2026-10-17 20:00:47 + [2026-10-16 12:13:51] iteration 1883/ 2384 | consumed samples: 60256 | elapsed time per iteration (ms): 220244.0 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 76175.6 | valid tokens/s: 76175.6 | physical tokens/s: 76984.0 | loss tokens/s: 58779.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.566153E-01 | lm-global-supervised-tokens: 1.294576E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.818662E-02 | lm-token-buffer-mib: 2.338928E+02 | global_load_balancing_loss: 1.000308E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11155.850 | params rms: 0.060405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:39:02.224867 | finish at 2026-10-17 18:52:53 + [2026-10-16 12:17:31] iteration 1884/ 2384 | consumed samples: 60288 | elapsed time per iteration (ms): 220881.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 75955.7 | valid tokens/s: 75955.7 | physical tokens/s: 76805.7 | loss tokens/s: 56531.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.516803E-01 | lm-global-supervised-tokens: 1.248672E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.503604E-02 | lm-token-buffer-mib: 2.345376E+02 | global_load_balancing_loss: 1.000705E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11155.819 | params rms: 0.060405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:40:40.528512 | finish at 2026-10-17 18:58:12 + [2026-10-16 12:21:12] iteration 1885/ 2384 | consumed samples: 60320 | elapsed time per iteration (ms): 220650.3 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.95% | FLOPs estimator: workload-aware | logical tokens/s: 76035.3 | valid tokens/s: 76035.3 | physical tokens/s: 76744.8 | loss tokens/s: 58961.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.514304E-01 | lm-global-supervised-tokens: 1.300996E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.637024E-02 | lm-token-buffer-mib: 2.348708E+02 | global_load_balancing_loss: 9.996489E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11155.789 | params rms: 0.060405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:35:04.500903 | finish at 2026-10-17 18:56:17 + [2026-10-16 12:24:57] iteration 1886/ 2384 | consumed samples: 60352 | elapsed time per iteration (ms): 224999.6 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 74565.5 | valid tokens/s: 74565.5 | physical tokens/s: 75762.8 | loss tokens/s: 58165.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.902153E-01 | lm-global-supervised-tokens: 1.308723E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.293940E-02 | lm-token-buffer-mib: 2.373205E+02 | global_load_balancing_loss: 1.000111E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11155.760 | params rms: 0.060405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:07:29.794830 | finish at 2026-10-17 19:32:27 + [2026-10-16 12:28:43] iteration 1887/ 2384 | consumed samples: 60384 | elapsed time per iteration (ms): 225539.2 | throughput per GPU (TFLOP/s/GPU): 27.1 | MFU 8.69% | FLOPs estimator: workload-aware | logical tokens/s: 74387.0 | valid tokens/s: 74387.0 | physical tokens/s: 74833.2 | loss tokens/s: 54839.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.520191E-01 | lm-global-supervised-tokens: 1.236849E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.752296E-02 | lm-token-buffer-mib: 2.339738E+02 | global_load_balancing_loss: 1.000554E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11155.733 | params rms: 0.060405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:08:13.006606 | finish at 2026-10-17 19:36:56 + [2026-10-16 12:32:21] iteration 1888/ 2384 | consumed samples: 60416 | elapsed time per iteration (ms): 218532.1 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 76772.2 | valid tokens/s: 76772.2 | physical tokens/s: 77370.0 | loss tokens/s: 58957.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.430929E-01 | lm-global-supervised-tokens: 1.288402E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.444339E-02 | lm-token-buffer-mib: 2.328550E+02 | global_load_balancing_loss: 9.990982E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11155.704 | params rms: 0.060405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:06:31.929504 | finish at 2026-10-17 18:38:53 + [2026-10-16 12:36:03] iteration 1889/ 2384 | consumed samples: 60448 | elapsed time per iteration (ms): 221585.6 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.21% | FLOPs estimator: workload-aware | logical tokens/s: 75714.2 | valid tokens/s: 75714.2 | physical tokens/s: 76231.7 | loss tokens/s: 60442.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.965471E-01 | lm-global-supervised-tokens: 1.339328E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.321926E-02 | lm-token-buffer-mib: 2.323961E+02 | global_load_balancing_loss: 9.990339E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11155.674 | params rms: 0.060404 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:28:04.867584 | finish at 2026-10-17 19:04:08 + [2026-10-16 12:40:06] iteration 1890/ 2384 | consumed samples: 60480 | elapsed time per iteration (ms): 243515.6 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 68895.8 | valid tokens/s: 68895.8 | physical tokens/s: 69931.1 | loss tokens/s: 51184.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.043231E-01 | lm-global-supervised-tokens: 1.246417E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.218910E-02 | lm-token-buffer-mib: 2.332724E+02 | global_load_balancing_loss: 1.000039E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11155.642 | params rms: 0.060404 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:24:56.692721 | finish at 2026-10-17 22:05:03 + [2026-10-16 12:43:51] iteration 1891/ 2384 | consumed samples: 60512 | elapsed time per iteration (ms): 224377.8 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.06% | FLOPs estimator: workload-aware | logical tokens/s: 74772.0 | valid tokens/s: 74772.0 | physical tokens/s: 75130.7 | loss tokens/s: 58518.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.143646E-01 | lm-global-supervised-tokens: 1.313017E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.270962E-02 | lm-token-buffer-mib: 2.325093E+02 | global_load_balancing_loss: 1.000407E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11155.609 | params rms: 0.060404 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:43:38.246108 | finish at 2026-10-17 19:27:29 + [2026-10-16 12:47:32] iteration 1892/ 2384 | consumed samples: 60544 | elapsed time per iteration (ms): 221639.2 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75695.8 | valid tokens/s: 75695.8 | physical tokens/s: 76207.8 | loss tokens/s: 58692.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.348968E-01 | lm-global-supervised-tokens: 1.300855E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.519378E-02 | lm-token-buffer-mib: 2.342342E+02 | global_load_balancing_loss: 1.000890E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11155.575 | params rms: 0.060404 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:17:26.479231 | finish at 2026-10-17 19:04:59 + [2026-10-16 12:51:14] iteration 1893/ 2384 | consumed samples: 60576 | elapsed time per iteration (ms): 221385.9 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75782.4 | valid tokens/s: 75782.4 | physical tokens/s: 76044.6 | loss tokens/s: 50735.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.843380E-01 | lm-global-supervised-tokens: 1.123218E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.603576E-02 | lm-token-buffer-mib: 2.328928E+02 | global_load_balancing_loss: 1.001428E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11155.544 | params rms: 0.060404 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:11:40.482646 | finish at 2026-10-17 19:02:54 + [2026-10-16 12:54:58] iteration 1894/ 2384 | consumed samples: 60608 | elapsed time per iteration (ms): 224107.3 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 74862.2 | valid tokens/s: 74862.2 | physical tokens/s: 75574.9 | loss tokens/s: 56512.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.585781E-01 | lm-global-supervised-tokens: 1.266494E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.491943E-02 | lm-token-buffer-mib: 2.342471E+02 | global_load_balancing_loss: 1.002302E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11155.514 | params rms: 0.060403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:30:12.592442 | finish at 2026-10-17 19:25:10 + [2026-10-16 12:58:39] iteration 1895/ 2384 | consumed samples: 60640 | elapsed time per iteration (ms): 221506.9 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 75741.1 | valid tokens/s: 75741.1 | physical tokens/s: 76483.0 | loss tokens/s: 56550.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.220423E-01 | lm-global-supervised-tokens: 1.252637E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.590452E-02 | lm-token-buffer-mib: 2.336807E+02 | global_load_balancing_loss: 1.000829E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11155.485 | params rms: 0.060403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:05:16.891740 | finish at 2026-10-17 19:03:56 + [2026-10-16 13:02:27] iteration 1896/ 2384 | consumed samples: 60672 | elapsed time per iteration (ms): 227274.3 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 73322.1 | valid tokens/s: 73322.1 | physical tokens/s: 74532.8 | loss tokens/s: 56543.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.777818E-01 | lm-global-supervised-tokens: 1.285080E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.978364E-02 | lm-token-buffer-mib: 2.337254E+02 | global_load_balancing_loss: 1.001147E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11155.454 | params rms: 0.060403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:48:29.864964 | finish at 2026-10-17 19:50:56 + [2026-10-16 13:06:09] iteration 1897/ 2384 | consumed samples: 60704 | elapsed time per iteration (ms): 222047.7 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 75556.6 | valid tokens/s: 75556.6 | physical tokens/s: 76352.0 | loss tokens/s: 57422.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.623351E-01 | lm-global-supervised-tokens: 1.275061E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.428804E-02 | lm-token-buffer-mib: 2.325727E+02 | global_load_balancing_loss: 9.992895E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11155.422 | params rms: 0.060403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:02:17.231026 | finish at 2026-10-17 19:08:26 + [2026-10-16 13:09:49] iteration 1898/ 2384 | consumed samples: 60736 | elapsed time per iteration (ms): 220432.7 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 76110.3 | valid tokens/s: 76110.2 | physical tokens/s: 76561.0 | loss tokens/s: 60273.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.181156E-01 | lm-global-supervised-tokens: 1.328620E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.420554E-02 | lm-token-buffer-mib: 2.340612E+02 | global_load_balancing_loss: 9.977868E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11155.392 | params rms: 0.060403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:45:30.284745 | finish at 2026-10-17 18:55:19 + [2026-10-16 13:13:29] iteration 1899/ 2384 | consumed samples: 60768 | elapsed time per iteration (ms): 219506.3 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.58% | FLOPs estimator: workload-aware | logical tokens/s: 76431.4 | valid tokens/s: 76431.4 | physical tokens/s: 77035.4 | loss tokens/s: 58544.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.289098E-01 | lm-global-supervised-tokens: 1.285095E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.999584E-02 | lm-token-buffer-mib: 2.327939E+02 | global_load_balancing_loss: 9.988712E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11155.363 | params rms: 0.060403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:34:20.574566 | finish at 2026-10-17 18:47:49 + [2026-10-16 13:17:32] iteration 1900/ 2384 | consumed samples: 60800 | elapsed time per iteration (ms): 243760.3 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 68826.6 | valid tokens/s: 68826.6 | physical tokens/s: 69327.9 | loss tokens/s: 51092.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.260226E-01 | lm-global-supervised-tokens: 1.245425E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.284792E-02 | lm-token-buffer-mib: 2.337204E+02 | global_load_balancing_loss: 9.990971E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11155.334 | params rms: 0.060403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:46:19.967045 | finish at 2026-10-17 22:03:52 + [2026-10-16 13:21:17] iteration 1901/ 2384 | consumed samples: 60832 | elapsed time per iteration (ms): 224559.8 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.91% | FLOPs estimator: workload-aware | logical tokens/s: 74711.4 | valid tokens/s: 74711.4 | physical tokens/s: 75235.1 | loss tokens/s: 53941.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.598559E-01 | lm-global-supervised-tokens: 1.211313E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.590185E-02 | lm-token-buffer-mib: 2.335574E+02 | global_load_balancing_loss: 1.000190E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11155.304 | params rms: 0.060402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:07:42.369538 | finish at 2026-10-17 19:28:59 + [2026-10-16 13:24:56] iteration 1902/ 2384 | consumed samples: 60864 | elapsed time per iteration (ms): 219338.3 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 76490.0 | valid tokens/s: 76490.0 | physical tokens/s: 77680.0 | loss tokens/s: 54959.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.333076E-01 | lm-global-supervised-tokens: 1.205466E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.767521E-02 | lm-token-buffer-mib: 2.334333E+02 | global_load_balancing_loss: 1.001457E+00 | loss scale: 1.0 | grad norm: 0.218 | params norm: 11155.272 | params rms: 0.060402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:22:01.083004 | finish at 2026-10-17 18:46:57 + [2026-10-16 13:28:38] iteration 1903/ 2384 | consumed samples: 60896 | elapsed time per iteration (ms): 221797.1 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 75642.1 | valid tokens/s: 75642.1 | physical tokens/s: 76132.4 | loss tokens/s: 55802.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.361286E-01 | lm-global-supervised-tokens: 1.237689E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.564441E-02 | lm-token-buffer-mib: 2.327645E+02 | global_load_balancing_loss: 1.002441E+00 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11155.241 | params rms: 0.060402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:38:04.426004 | finish at 2026-10-17 19:06:42 + [2026-10-16 13:32:24] iteration 1904/ 2384 | consumed samples: 60928 | elapsed time per iteration (ms): 226142.2 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 74053.2 | valid tokens/s: 74053.2 | physical tokens/s: 74775.2 | loss tokens/s: 57406.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.014263E-01 | lm-global-supervised-tokens: 1.298211E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.289868E-02 | lm-token-buffer-mib: 2.345094E+02 | global_load_balancing_loss: 1.001321E+00 | loss scale: 1.0 | grad norm: 0.387 | params norm: 11155.212 | params rms: 0.060402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:09:08.232651 | finish at 2026-10-17 19:41:32 + [2026-10-16 13:36:07] iteration 1905/ 2384 | consumed samples: 60960 | elapsed time per iteration (ms): 223083.5 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 75205.9 | valid tokens/s: 75205.9 | physical tokens/s: 75967.7 | loss tokens/s: 50121.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.662739E-01 | lm-global-supervised-tokens: 1.118136E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.627808E-02 | lm-token-buffer-mib: 2.333212E+02 | global_load_balancing_loss: 1.001493E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11155.181 | params rms: 0.060402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:40:57.020324 | finish at 2026-10-17 19:17:04 + [2026-10-16 13:39:49] iteration 1906/ 2384 | consumed samples: 60992 | elapsed time per iteration (ms): 221490.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75746.8 | valid tokens/s: 75746.8 | physical tokens/s: 76329.3 | loss tokens/s: 52040.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.070637E-01 | lm-global-supervised-tokens: 1.152647E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.916865E-02 | lm-token-buffer-mib: 2.327356E+02 | global_load_balancing_loss: 1.000005E+00 | loss scale: 1.0 | grad norm: 0.238 | params norm: 11155.151 | params rms: 0.060402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:24:32.576484 | finish at 2026-10-17 19:04:21 + [2026-10-16 13:43:33] iteration 1907/ 2384 | consumed samples: 61024 | elapsed time per iteration (ms): 224702.9 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 74664.0 | valid tokens/s: 74664.0 | physical tokens/s: 75266.1 | loss tokens/s: 54794.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.797282E-01 | lm-global-supervised-tokens: 1.231236E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.209049E-02 | lm-token-buffer-mib: 2.350680E+02 | global_load_balancing_loss: 9.991000E-01 | loss scale: 1.0 | grad norm: 0.333 | params norm: 11155.122 | params rms: 0.060401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:46:23.275307 | finish at 2026-10-17 19:29:57 + [2026-10-16 13:47:17] iteration 1908/ 2384 | consumed samples: 61056 | elapsed time per iteration (ms): 223304.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 75131.5 | valid tokens/s: 75131.5 | physical tokens/s: 76107.7 | loss tokens/s: 54855.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.824656E-01 | lm-global-supervised-tokens: 1.224952E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.675312E-02 | lm-token-buffer-mib: 2.347160E+02 | global_load_balancing_loss: 1.000069E+00 | loss scale: 1.0 | grad norm: 0.252 | params norm: 11155.091 | params rms: 0.060401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:31:32.937963 | finish at 2026-10-17 19:18:50 + [2026-10-16 13:51:05] iteration 1909/ 2384 | consumed samples: 61088 | elapsed time per iteration (ms): 227818.6 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 73642.7 | valid tokens/s: 73642.7 | physical tokens/s: 75058.7 | loss tokens/s: 59063.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.265426E-01 | lm-global-supervised-tokens: 1.345569E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.414818E-02 | lm-token-buffer-mib: 2.367993E+02 | global_load_balancing_loss: 1.002383E+00 | loss scale: 1.0 | grad norm: 0.388 | params norm: 11155.058 | params rms: 0.060401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:03:33.813907 | finish at 2026-10-17 19:54:38 + [2026-10-16 13:55:09] iteration 1910/ 2384 | consumed samples: 61120 | elapsed time per iteration (ms): 244109.4 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 68727.9 | valid tokens/s: 68727.9 | physical tokens/s: 69145.9 | loss tokens/s: 51476.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.680138E-01 | lm-global-supervised-tokens: 1.256600E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.666241E-02 | lm-token-buffer-mib: 2.333857E+02 | global_load_balancing_loss: 1.000584E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11155.030 | params rms: 0.060401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:08:27.867708 | finish at 2026-10-17 22:03:37 + [2026-10-16 13:58:56] iteration 1911/ 2384 | consumed samples: 61152 | elapsed time per iteration (ms): 227315.8 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 73805.6 | valid tokens/s: 73805.6 | physical tokens/s: 74316.5 | loss tokens/s: 54259.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.070041E-01 | lm-global-supervised-tokens: 1.233401E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.436369E-02 | lm-token-buffer-mib: 2.345091E+02 | global_load_balancing_loss: 1.000645E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11155.004 | params rms: 0.060401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:52:00.384993 | finish at 2026-10-17 19:50:56 + [2026-10-16 14:02:36] iteration 1912/ 2384 | consumed samples: 61184 | elapsed time per iteration (ms): 219674.6 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 76372.9 | valid tokens/s: 76372.9 | physical tokens/s: 76754.1 | loss tokens/s: 56869.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.726805E-01 | lm-global-supervised-tokens: 1.249276E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.155299E-02 | lm-token-buffer-mib: 2.322708E+02 | global_load_balancing_loss: 1.000170E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11154.974 | params rms: 0.060401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:48:06.415873 | finish at 2026-10-17 18:50:42 + [2026-10-16 14:06:17] iteration 1913/ 2384 | consumed samples: 61216 | elapsed time per iteration (ms): 221866.8 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 75618.2 | valid tokens/s: 75618.2 | physical tokens/s: 76126.4 | loss tokens/s: 53002.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.070940E-01 | lm-global-supervised-tokens: 1.175937E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.247718E-02 | lm-token-buffer-mib: 2.345566E+02 | global_load_balancing_loss: 9.988826E-01 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11154.940 | params rms: 0.060400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:01:39.249807 | finish at 2026-10-17 19:07:57 + [2026-10-16 14:09:56] iteration 1914/ 2384 | consumed samples: 61248 | elapsed time per iteration (ms): 218296.6 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 76855.1 | valid tokens/s: 76855.1 | physical tokens/s: 77172.9 | loss tokens/s: 55514.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.566443E-01 | lm-global-supervised-tokens: 1.211857E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.438929E-02 | lm-token-buffer-mib: 2.329854E+02 | global_load_balancing_loss: 9.995880E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11154.905 | params rms: 0.060400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:29:59.406531 | finish at 2026-10-17 18:39:55 + [2026-10-16 14:13:40] iteration 1915/ 2384 | consumed samples: 61280 | elapsed time per iteration (ms): 223859.4 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 74945.2 | valid tokens/s: 74945.2 | physical tokens/s: 75473.7 | loss tokens/s: 54618.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.108145E-01 | lm-global-supervised-tokens: 1.222686E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.377782E-02 | lm-token-buffer-mib: 2.337812E+02 | global_load_balancing_loss: 1.001135E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11154.873 | params rms: 0.060400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:09:50.057833 | finish at 2026-10-17 19:23:30 + [2026-10-16 14:17:20] iteration 1916/ 2384 | consumed samples: 61312 | elapsed time per iteration (ms): 220300.2 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.44% | FLOPs estimator: workload-aware | logical tokens/s: 76156.1 | valid tokens/s: 76156.1 | physical tokens/s: 76895.8 | loss tokens/s: 57429.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.773759E-01 | lm-global-supervised-tokens: 1.265177E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.831144E-02 | lm-token-buffer-mib: 2.339507E+02 | global_load_balancing_loss: 1.001457E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11154.843 | params rms: 0.060400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:38:20.514459 | finish at 2026-10-17 18:55:40 + [2026-10-16 14:21:02] iteration 1917/ 2384 | consumed samples: 61344 | elapsed time per iteration (ms): 222272.7 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75480.3 | valid tokens/s: 75480.3 | physical tokens/s: 75855.2 | loss tokens/s: 52074.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.579583E-01 | lm-global-supervised-tokens: 1.157476E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.188859E-02 | lm-token-buffer-mib: 2.326290E+02 | global_load_balancing_loss: 1.002099E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11154.810 | params rms: 0.060400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:50:01.331115 | finish at 2026-10-17 19:11:04 + [2026-10-16 14:24:46] iteration 1918/ 2384 | consumed samples: 61376 | elapsed time per iteration (ms): 223715.1 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 74993.5 | valid tokens/s: 74993.5 | physical tokens/s: 75559.5 | loss tokens/s: 56091.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.700852E-01 | lm-global-supervised-tokens: 1.254851E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.147783E-02 | lm-token-buffer-mib: 2.345804E+02 | global_load_balancing_loss: 1.000975E+00 | loss scale: 1.0 | grad norm: 0.196 | params norm: 11154.780 | params rms: 0.060400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:57:31.249178 | finish at 2026-10-17 19:22:17 + [2026-10-16 14:28:28] iteration 1919/ 2384 | consumed samples: 61408 | elapsed time per iteration (ms): 221591.4 | throughput per GPU (TFLOP/s/GPU): 25.3 | MFU 8.09% | FLOPs estimator: workload-aware | logical tokens/s: 75712.3 | valid tokens/s: 75712.3 | physical tokens/s: 76157.2 | loss tokens/s: 51911.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.475169E-01 | lm-global-supervised-tokens: 1.150317E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.414265E-02 | lm-token-buffer-mib: 2.336403E+02 | global_load_balancing_loss: 1.000774E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11154.751 | params rms: 0.060399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:37:20.023795 | finish at 2026-10-17 19:05:48 + [2026-10-16 14:32:35] iteration 1920/ 2384 | consumed samples: 61440 | elapsed time per iteration (ms): 247375.9 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 67820.7 | valid tokens/s: 67820.7 | physical tokens/s: 68561.4 | loss tokens/s: 52671.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.966212E-01 | lm-global-supervised-tokens: 1.302972E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.356033E-02 | lm-token-buffer-mib: 2.356852E+02 | global_load_balancing_loss: 9.995731E-01 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11154.721 | params rms: 0.060399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:53:02.414627 | finish at 2026-10-17 22:25:37 +times across ranks (ms): + save-checkpoint: + rank 0: 24036.11 + rank 1: 24036.12 + rank 2: 24036.13 + rank 3: 24036.11 + rank 4: 24036.12 + rank 5: 24036.13 + rank 6: 24036.13 + rank 7: 24036.14 + rank 8: 24035.95 + rank 9: 24035.99 + rank 10: 24035.95 + rank 11: 24035.95 + rank 12: 24036.03 + rank 13: 24035.93 + rank 14: 24035.95 + rank 15: 24035.95 + rank 16: 24035.98 + rank 17: 24035.97 + rank 18: 24035.97 + rank 19: 24036.02 + rank 20: 24035.95 + rank 21: 24035.95 + rank 22: 24036.00 + rank 23: 24035.96 + rank 24: 24035.76 + rank 25: 24035.57 + rank 26: 24035.81 + rank 27: 24035.60 + rank 28: 24035.56 + rank 29: 24035.59 + rank 30: 24035.57 + rank 31: 24035.58 + rank 32: 24037.08 + rank 33: 24037.08 + rank 34: 24037.04 + rank 35: 24037.07 + rank 36: 24037.03 + rank 37: 24037.09 + rank 38: 24037.02 + rank 39: 24037.05 + rank 40: 24037.41 + rank 41: 24037.32 + rank 42: 24037.32 + rank 43: 24037.35 + rank 44: 24037.29 + rank 45: 24037.32 + rank 46: 24037.31 + rank 47: 24037.29 + rank 48: 24036.49 + rank 49: 24036.52 + rank 50: 24036.54 + rank 51: 24036.50 + rank 52: 24036.47 + rank 53: 24036.51 + rank 54: 24036.47 + rank 55: 24036.49 + rank 56: 24036.61 + rank 57: 24036.59 + rank 58: 24036.60 + rank 59: 24036.59 + rank 60: 24036.59 + rank 61: 24036.58 + rank 62: 24036.56 + rank 63: 24036.57 + [2026-10-16 14:36:41] iteration 1921/ 2384 | consumed samples: 61472 | elapsed time per iteration (ms): 222070.4 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.81% | FLOPs estimator: workload-aware | logical tokens/s: 75548.9 | valid tokens/s: 75548.9 | physical tokens/s: 76141.6 | loss tokens/s: 60596.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.115759E-01 | lm-global-supervised-tokens: 1.345674E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.298804E-02 | lm-token-buffer-mib: 2.326775E+02 | global_load_balancing_loss: 9.994186E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11154.687 | params rms: 0.060399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:33:38.603369 | finish at 2026-10-17 19:10:20 + [2026-10-16 14:40:28] iteration 1922/ 2384 | consumed samples: 61504 | elapsed time per iteration (ms): 227279.2 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.02% | FLOPs estimator: workload-aware | logical tokens/s: 73817.7 | valid tokens/s: 73817.7 | physical tokens/s: 74200.6 | loss tokens/s: 57645.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.375400E-01 | lm-global-supervised-tokens: 1.310160E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.545688E-02 | lm-token-buffer-mib: 2.335052E+02 | global_load_balancing_loss: 1.002036E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11154.652 | params rms: 0.060399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:10:02.986030 | finish at 2026-10-17 19:50:31 + [2026-10-16 14:44:09] iteration 1923/ 2384 | consumed samples: 61536 | elapsed time per iteration (ms): 220671.2 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 76028.0 | valid tokens/s: 76028.0 | physical tokens/s: 76890.4 | loss tokens/s: 62178.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.901527E-01 | lm-global-supervised-tokens: 1.372102E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.548064E-02 | lm-token-buffer-mib: 2.327388E+02 | global_load_balancing_loss: 9.997468E-01 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11154.619 | params rms: 0.060399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:15:29.405192 | finish at 2026-10-17 18:59:38 + [2026-10-16 14:47:51] iteration 1924/ 2384 | consumed samples: 61568 | elapsed time per iteration (ms): 221608.8 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75706.2 | valid tokens/s: 75706.2 | physical tokens/s: 77001.7 | loss tokens/s: 61481.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.918244E-01 | lm-global-supervised-tokens: 1.362493E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.564422E-02 | lm-token-buffer-mib: 2.362472E+02 | global_load_balancing_loss: 9.992981E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11154.589 | params rms: 0.060398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:19:00.033932 | finish at 2026-10-17 19:06:51 + [2026-10-16 14:51:31] iteration 1925/ 2384 | consumed samples: 61600 | elapsed time per iteration (ms): 219916.4 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.28% | FLOPs estimator: workload-aware | logical tokens/s: 76289.0 | valid tokens/s: 76289.0 | physical tokens/s: 77155.1 | loss tokens/s: 59532.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.446249E-01 | lm-global-supervised-tokens: 1.309216E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.192056E-02 | lm-token-buffer-mib: 2.337920E+02 | global_load_balancing_loss: 9.995452E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11154.558 | params rms: 0.060398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:02:21.635021 | finish at 2026-10-17 18:53:52 + [2026-10-16 14:55:16] iteration 1926/ 2384 | consumed samples: 61632 | elapsed time per iteration (ms): 225085.6 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.96% | FLOPs estimator: workload-aware | logical tokens/s: 74536.9 | valid tokens/s: 74536.9 | physical tokens/s: 75542.7 | loss tokens/s: 58141.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.165924E-01 | lm-global-supervised-tokens: 1.308690E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.779066E-02 | lm-token-buffer-mib: 2.345936E+02 | global_load_balancing_loss: 9.999164E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11154.522 | params rms: 0.060398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:38:09.198967 | finish at 2026-10-17 19:33:25 + [2026-10-16 14:58:56] iteration 1927/ 2384 | consumed samples: 61664 | elapsed time per iteration (ms): 220565.1 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 76064.5 | valid tokens/s: 76064.5 | physical tokens/s: 76744.5 | loss tokens/s: 59682.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.765848E-01 | lm-global-supervised-tokens: 1.316394E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.959996E-02 | lm-token-buffer-mib: 2.341419E+02 | global_load_balancing_loss: 9.995072E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11154.486 | params rms: 0.060398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:59:58.264299 | finish at 2026-10-17 18:58:55 + [2026-10-16 15:02:40] iteration 1928/ 2384 | consumed samples: 61696 | elapsed time per iteration (ms): 223431.4 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.63% | FLOPs estimator: workload-aware | logical tokens/s: 75088.7 | valid tokens/s: 75088.7 | physical tokens/s: 75929.6 | loss tokens/s: 59259.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.067478E-01 | lm-global-supervised-tokens: 1.324040E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.239530E-02 | lm-token-buffer-mib: 2.334358E+02 | global_load_balancing_loss: 1.000491E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11154.449 | params rms: 0.060398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:18:04.732344 | finish at 2026-10-17 19:20:44 + [2026-10-16 15:06:24] iteration 1929/ 2384 | consumed samples: 61728 | elapsed time per iteration (ms): 223807.1 | throughput per GPU (TFLOP/s/GPU): 26.3 | MFU 8.43% | FLOPs estimator: workload-aware | logical tokens/s: 74962.8 | valid tokens/s: 74962.7 | physical tokens/s: 75782.7 | loss tokens/s: 54485.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.844590E-01 | lm-global-supervised-tokens: 1.219419E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.571198E-02 | lm-token-buffer-mib: 2.345739E+02 | global_load_balancing_loss: 1.000341E+00 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11154.412 | params rms: 0.060398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:17:12.215014 | finish at 2026-10-17 19:23:36 + [2026-10-16 15:10:26] iteration 1930/ 2384 | consumed samples: 61760 | elapsed time per iteration (ms): 242699.6 | throughput per GPU (TFLOP/s/GPU): 17.9 | MFU 5.73% | FLOPs estimator: workload-aware | logical tokens/s: 69127.4 | valid tokens/s: 69127.4 | physical tokens/s: 70000.3 | loss tokens/s: 51738.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.332217E-01 | lm-global-supervised-tokens: 1.255692E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.516776E-02 | lm-token-buffer-mib: 2.348447E+02 | global_load_balancing_loss: 1.001099E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11154.374 | params rms: 0.060397 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:36:25.640256 | finish at 2026-10-17 21:46:52 + [2026-10-16 15:14:08] iteration 1931/ 2384 | consumed samples: 61792 | elapsed time per iteration (ms): 221970.1 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75583.1 | valid tokens/s: 75583.1 | physical tokens/s: 76211.8 | loss tokens/s: 58090.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.393806E-01 | lm-global-supervised-tokens: 1.289429E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.698390E-02 | lm-token-buffer-mib: 2.337276E+02 | global_load_balancing_loss: 1.001859E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11154.337 | params rms: 0.060397 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:55:52.448030 | finish at 2026-10-17 19:10:01 + [2026-10-16 15:17:53] iteration 1932/ 2384 | consumed samples: 61824 | elapsed time per iteration (ms): 225040.6 | throughput per GPU (TFLOP/s/GPU): 18.2 | MFU 5.84% | FLOPs estimator: workload-aware | logical tokens/s: 74551.9 | valid tokens/s: 74551.9 | physical tokens/s: 75303.0 | loss tokens/s: 57527.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.780935E-01 | lm-global-supervised-tokens: 1.294600E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.761037E-02 | lm-token-buffer-mib: 2.337162E+02 | global_load_balancing_loss: 1.001265E+00 | loss scale: 1.0 | grad norm: 0.301 | params norm: 11154.302 | params rms: 0.060397 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:15:18.334632 | finish at 2026-10-17 19:33:12 + [2026-10-16 15:21:39] iteration 1933/ 2384 | consumed samples: 61856 | elapsed time per iteration (ms): 225748.9 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 74317.8 | valid tokens/s: 74317.8 | physical tokens/s: 75534.5 | loss tokens/s: 53574.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.072748E-01 | lm-global-supervised-tokens: 1.209445E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.660406E-02 | lm-token-buffer-mib: 2.350245E+02 | global_load_balancing_loss: 1.000552E+00 | loss scale: 1.0 | grad norm: 0.464 | params norm: 11154.268 | params rms: 0.060397 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:16:52.764409 | finish at 2026-10-17 19:38:32 + [2026-10-16 15:25:22] iteration 1934/ 2384 | consumed samples: 61888 | elapsed time per iteration (ms): 223388.2 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 75103.2 | valid tokens/s: 75103.2 | physical tokens/s: 75822.5 | loss tokens/s: 59466.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.763932E-01 | lm-global-supervised-tokens: 1.328410E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.430831E-02 | lm-token-buffer-mib: 2.334579E+02 | global_load_balancing_loss: 1.000221E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11154.233 | params rms: 0.060397 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:55:24.705899 | finish at 2026-10-17 19:20:47 + [2026-10-16 15:29:05] iteration 1935/ 2384 | consumed samples: 61920 | elapsed time per iteration (ms): 222378.9 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 75444.0 | valid tokens/s: 75444.0 | physical tokens/s: 75941.6 | loss tokens/s: 52914.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.889365E-01 | lm-global-supervised-tokens: 1.176705E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.012388E-02 | lm-token-buffer-mib: 2.332072E+02 | global_load_balancing_loss: 9.997922E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11154.194 | params rms: 0.060396 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:44:08.118951 | finish at 2026-10-17 19:13:13 + [2026-10-16 15:32:49] iteration 1936/ 2384 | consumed samples: 61952 | elapsed time per iteration (ms): 224212.7 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 74827.1 | valid tokens/s: 74827.1 | physical tokens/s: 75461.2 | loss tokens/s: 55584.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.575691E-01 | lm-global-supervised-tokens: 1.246270E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.773133E-02 | lm-token-buffer-mib: 2.327951E+02 | global_load_balancing_loss: 9.986623E-01 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11154.156 | params rms: 0.060396 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:54:07.286667 | finish at 2026-10-17 19:26:56 + [2026-10-16 15:36:30] iteration 1937/ 2384 | consumed samples: 61984 | elapsed time per iteration (ms): 221320.0 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 75805.2 | valid tokens/s: 75805.1 | physical tokens/s: 76580.8 | loss tokens/s: 55661.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.120797E-01 | lm-global-supervised-tokens: 1.231906E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.357793E-02 | lm-token-buffer-mib: 2.344339E+02 | global_load_balancing_loss: 9.999555E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11154.118 | params rms: 0.060396 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:28:50.060858 | finish at 2026-10-17 19:05:20 + [2026-10-16 15:40:11] iteration 1938/ 2384 | consumed samples: 62016 | elapsed time per iteration (ms): 221114.6 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 75875.5 | valid tokens/s: 75875.5 | physical tokens/s: 76405.6 | loss tokens/s: 59672.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.389888E-01 | lm-global-supervised-tokens: 1.319439E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.332242E-02 | lm-token-buffer-mib: 2.324018E+02 | global_load_balancing_loss: 1.000782E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11154.083 | params rms: 0.060396 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:23:37.113489 | finish at 2026-10-17 19:03:48 + [2026-10-16 15:43:53] iteration 1939/ 2384 | consumed samples: 62048 | elapsed time per iteration (ms): 221728.6 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.39% | FLOPs estimator: workload-aware | logical tokens/s: 75665.4 | valid tokens/s: 75665.4 | physical tokens/s: 76751.1 | loss tokens/s: 53366.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.798789E-01 | lm-global-supervised-tokens: 1.183278E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.613904E-02 | lm-token-buffer-mib: 2.351981E+02 | global_load_balancing_loss: 1.001660E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11154.050 | params rms: 0.060396 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:24:29.231892 | finish at 2026-10-17 19:08:22 + [2026-10-16 15:47:56] iteration 1940/ 2384 | consumed samples: 62080 | elapsed time per iteration (ms): 242819.5 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 69093.3 | valid tokens/s: 69093.3 | physical tokens/s: 69779.4 | loss tokens/s: 52260.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.289542E-01 | lm-global-supervised-tokens: 1.268984E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.078355E-02 | lm-token-buffer-mib: 2.332166E+02 | global_load_balancing_loss: 1.002274E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11154.016 | params rms: 0.060395 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:56:51.852694 | finish at 2026-10-17 21:44:48 + [2026-10-16 15:51:37] iteration 1941/ 2384 | consumed samples: 62112 | elapsed time per iteration (ms): 221431.2 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 75767.1 | valid tokens/s: 75767.1 | physical tokens/s: 76630.5 | loss tokens/s: 61518.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.102042E-01 | lm-global-supervised-tokens: 1.362208E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.636823E-02 | lm-token-buffer-mib: 2.351321E+02 | global_load_balancing_loss: 1.001275E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11153.982 | params rms: 0.060395 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:14:54.006297 | finish at 2026-10-17 19:06:31 + [2026-10-16 15:55:19] iteration 1942/ 2384 | consumed samples: 62144 | elapsed time per iteration (ms): 221491.6 | throughput per GPU (TFLOP/s/GPU): 25.4 | MFU 8.15% | FLOPs estimator: workload-aware | logical tokens/s: 75746.4 | valid tokens/s: 75746.4 | physical tokens/s: 76119.8 | loss tokens/s: 53506.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.861991E-01 | lm-global-supervised-tokens: 1.185118E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.220028E-02 | lm-token-buffer-mib: 2.328828E+02 | global_load_balancing_loss: 9.994562E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11153.950 | params rms: 0.060395 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:11:39.287111 | finish at 2026-10-17 19:06:58 + [2026-10-16 15:59:01] iteration 1943/ 2384 | consumed samples: 62176 | elapsed time per iteration (ms): 222338.9 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 75457.6 | valid tokens/s: 75457.6 | physical tokens/s: 76354.5 | loss tokens/s: 58081.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.177632E-01 | lm-global-supervised-tokens: 1.291380E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.272308E-02 | lm-token-buffer-mib: 2.339918E+02 | global_load_balancing_loss: 9.996463E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11153.917 | params rms: 0.060395 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:14:11.475758 | finish at 2026-10-17 19:13:13 + [2026-10-16 16:02:43] iteration 1944/ 2384 | consumed samples: 62208 | elapsed time per iteration (ms): 221578.1 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 75716.7 | valid tokens/s: 75716.7 | physical tokens/s: 76406.7 | loss tokens/s: 59309.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.475143E-01 | lm-global-supervised-tokens: 1.314173E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.763226E-02 | lm-token-buffer-mib: 2.329520E+02 | global_load_balancing_loss: 1.000053E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11153.882 | params rms: 0.060395 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:04:54.350872 | finish at 2026-10-17 19:07:37 + [2026-10-16 16:06:25] iteration 1945/ 2384 | consumed samples: 62240 | elapsed time per iteration (ms): 222358.0 | throughput per GPU (TFLOP/s/GPU): 16.7 | MFU 5.37% | FLOPs estimator: workload-aware | logical tokens/s: 75451.3 | valid tokens/s: 75451.2 | physical tokens/s: 76598.1 | loss tokens/s: 60039.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.073885E-01 | lm-global-supervised-tokens: 1.335033E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.699762E-02 | lm-token-buffer-mib: 2.362674E+02 | global_load_balancing_loss: 9.995694E-01 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11153.851 | params rms: 0.060394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:06:55.157517 | finish at 2026-10-17 19:13:20 + [2026-10-16 16:10:08] iteration 1946/ 2384 | consumed samples: 62272 | elapsed time per iteration (ms): 222748.2 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 75319.1 | valid tokens/s: 75319.1 | physical tokens/s: 76179.7 | loss tokens/s: 56215.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.842621E-01 | lm-global-supervised-tokens: 1.252193E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.103571E-02 | lm-token-buffer-mib: 2.341769E+02 | global_load_balancing_loss: 1.001239E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11153.819 | params rms: 0.060394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:06:03.706143 | finish at 2026-10-17 19:16:12 + [2026-10-16 16:13:54] iteration 1947/ 2384 | consumed samples: 62304 | elapsed time per iteration (ms): 225748.8 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 74318.0 | valid tokens/s: 74318.0 | physical tokens/s: 75361.9 | loss tokens/s: 57600.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.896113E-01 | lm-global-supervised-tokens: 1.300320E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.628457E-02 | lm-token-buffer-mib: 2.355405E+02 | global_load_balancing_loss: 1.000996E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11153.785 | params rms: 0.060394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:24:12.224888 | finish at 2026-10-17 19:38:06 + [2026-10-16 16:17:36] iteration 1948/ 2384 | consumed samples: 62336 | elapsed time per iteration (ms): 221899.5 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.48% | FLOPs estimator: workload-aware | logical tokens/s: 75607.1 | valid tokens/s: 75607.1 | physical tokens/s: 76042.2 | loss tokens/s: 58578.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.881292E-01 | lm-global-supervised-tokens: 1.299864E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.337527E-02 | lm-token-buffer-mib: 2.341586E+02 | global_load_balancing_loss: 1.002183E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11153.751 | params rms: 0.060394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:52:28.201808 | finish at 2026-10-17 19:10:04 + [2026-10-16 16:21:22] iteration 1949/ 2384 | consumed samples: 62368 | elapsed time per iteration (ms): 226365.1 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.91% | FLOPs estimator: workload-aware | logical tokens/s: 74115.6 | valid tokens/s: 74115.6 | physical tokens/s: 74902.0 | loss tokens/s: 57103.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.517172E-01 | lm-global-supervised-tokens: 1.292614E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.906264E-02 | lm-token-buffer-mib: 2.360646E+02 | global_load_balancing_loss: 1.001334E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11153.719 | params rms: 0.060394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:21:08.819497 | finish at 2026-10-17 19:42:31 + [2026-10-16 16:25:30] iteration 1950/ 2384 | consumed samples: 62400 | elapsed time per iteration (ms): 248507.2 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 67511.9 | valid tokens/s: 67511.9 | physical tokens/s: 68303.4 | loss tokens/s: 55283.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.740496E-01 | lm-global-supervised-tokens: 1.373827E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.148971E-02 | lm-token-buffer-mib: 2.360676E+02 | global_load_balancing_loss: 1.000625E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11153.692 | params rms: 0.060394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:57:32.120352 | finish at 2026-10-17 22:23:03 + [2026-10-16 16:29:13] iteration 1951/ 2384 | consumed samples: 62432 | elapsed time per iteration (ms): 222496.5 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.29% | FLOPs estimator: workload-aware | logical tokens/s: 75404.3 | valid tokens/s: 75404.3 | physical tokens/s: 75892.8 | loss tokens/s: 55481.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.681327E-01 | lm-global-supervised-tokens: 1.234433E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.202843E-02 | lm-token-buffer-mib: 2.329458E+02 | global_load_balancing_loss: 1.000087E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11153.666 | params rms: 0.060393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:45:40.982442 | finish at 2026-10-17 19:14:54 + [2026-10-16 16:32:53] iteration 1952/ 2384 | consumed samples: 62464 | elapsed time per iteration (ms): 220140.7 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 76211.3 | valid tokens/s: 76211.3 | physical tokens/s: 77103.0 | loss tokens/s: 56085.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.061705E-01 | lm-global-supervised-tokens: 1.234660E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.931657E-02 | lm-token-buffer-mib: 2.354854E+02 | global_load_balancing_loss: 9.987323E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11153.637 | params rms: 0.060393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:25:00.775337 | finish at 2026-10-17 18:57:54 + [2026-10-16 16:36:38] iteration 1953/ 2384 | consumed samples: 62496 | elapsed time per iteration (ms): 224699.8 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 74665.0 | valid tokens/s: 74665.0 | physical tokens/s: 75445.7 | loss tokens/s: 57160.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.233160E-01 | lm-global-supervised-tokens: 1.284384E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.484610E-02 | lm-token-buffer-mib: 2.343724E+02 | global_load_balancing_loss: 9.994987E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11153.608 | params rms: 0.060393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:54:05.612265 | finish at 2026-10-17 19:30:43 + [2026-10-16 16:40:18] iteration 1954/ 2384 | consumed samples: 62528 | elapsed time per iteration (ms): 220190.7 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 76194.0 | valid tokens/s: 76194.0 | physical tokens/s: 77095.4 | loss tokens/s: 59593.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.443402E-01 | lm-global-supervised-tokens: 1.312182E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.837821E-02 | lm-token-buffer-mib: 2.334017E+02 | global_load_balancing_loss: 1.000445E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11153.582 | params rms: 0.060393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:18:01.995282 | finish at 2026-10-17 18:58:20 + [2026-10-16 16:43:59] iteration 1955/ 2384 | consumed samples: 62560 | elapsed time per iteration (ms): 220748.8 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 76001.2 | valid tokens/s: 76001.2 | physical tokens/s: 77148.3 | loss tokens/s: 58523.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.000381E-01 | lm-global-supervised-tokens: 1.291896E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.307758E-02 | lm-token-buffer-mib: 2.325320E+02 | global_load_balancing_loss: 1.000583E+00 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11153.555 | params rms: 0.060393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:18:21.244422 | finish at 2026-10-17 19:02:20 + [2026-10-16 16:47:43] iteration 1956/ 2384 | consumed samples: 62592 | elapsed time per iteration (ms): 224225.4 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 74822.8 | valid tokens/s: 74822.8 | physical tokens/s: 75761.5 | loss tokens/s: 60314.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.641169E-01 | lm-global-supervised-tokens: 1.352411E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.403802E-02 | lm-token-buffer-mib: 2.343831E+02 | global_load_balancing_loss: 1.001222E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11153.530 | params rms: 0.060393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:39:28.465983 | finish at 2026-10-17 19:27:11 + [2026-10-16 16:51:23] iteration 1957/ 2384 | consumed samples: 62624 | elapsed time per iteration (ms): 220405.9 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 76119.4 | valid tokens/s: 76119.4 | physical tokens/s: 76917.6 | loss tokens/s: 52665.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.958975E-01 | lm-global-supervised-tokens: 1.160780E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.440594E-02 | lm-token-buffer-mib: 2.341848E+02 | global_load_balancing_loss: 1.001120E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11153.501 | params rms: 0.060393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:08:33.325409 | finish at 2026-10-17 18:59:57 + [2026-10-16 16:55:04] iteration 1958/ 2384 | consumed samples: 62656 | elapsed time per iteration (ms): 220331.1 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 76145.5 | valid tokens/s: 76145.5 | physical tokens/s: 76747.6 | loss tokens/s: 57063.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.719452E-01 | lm-global-supervised-tokens: 1.257294E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.520390E-02 | lm-token-buffer-mib: 2.332434E+02 | global_load_balancing_loss: 1.000794E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11153.470 | params rms: 0.060392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:04:21.033258 | finish at 2026-10-17 18:59:25 + [2026-10-16 16:58:50] iteration 1959/ 2384 | consumed samples: 62688 | elapsed time per iteration (ms): 226485.4 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 74076.3 | valid tokens/s: 74076.3 | physical tokens/s: 74882.6 | loss tokens/s: 54596.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.679428E-01 | lm-global-supervised-tokens: 1.236536E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.228946E-02 | lm-token-buffer-mib: 2.365630E+02 | global_load_balancing_loss: 1.001430E+00 | loss scale: 1.0 | grad norm: 0.248 | params norm: 11153.441 | params rms: 0.060392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:44:16.286877 | finish at 2026-10-17 19:43:06 + [2026-10-16 17:02:56] iteration 1960/ 2384 | consumed samples: 62720 | elapsed time per iteration (ms): 246084.3 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 68176.6 | valid tokens/s: 68176.6 | physical tokens/s: 68522.8 | loss tokens/s: 49711.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.017786E-01 | lm-global-supervised-tokens: 1.223313E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.299146E-02 | lm-token-buffer-mib: 2.339022E+02 | global_load_balancing_loss: 1.000285E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11153.415 | params rms: 0.060392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:58:59.758987 | finish at 2026-10-17 22:01:56 +times across ranks (ms): + save-checkpoint: + rank 0: 26029.44 + rank 1: 26029.46 + rank 2: 26029.45 + rank 3: 26029.46 + rank 4: 26029.40 + rank 5: 26029.46 + rank 6: 26029.43 + rank 7: 26029.44 + rank 8: 26028.87 + rank 9: 26028.88 + rank 10: 26028.89 + rank 11: 26028.86 + rank 12: 26028.88 + rank 13: 26028.85 + rank 14: 26028.87 + rank 15: 26029.24 + rank 16: 26030.11 + rank 17: 26030.11 + rank 18: 26029.95 + rank 19: 26030.12 + rank 20: 26030.10 + rank 21: 26030.09 + rank 22: 26029.97 + rank 23: 26030.13 + rank 24: 26029.01 + rank 25: 26029.00 + rank 26: 26029.01 + rank 27: 26028.96 + rank 28: 26029.28 + rank 29: 26029.11 + rank 30: 26028.89 + rank 31: 26028.96 + rank 32: 26029.21 + rank 33: 26029.22 + rank 34: 26029.27 + rank 35: 26029.25 + rank 36: 26029.29 + rank 37: 26029.20 + rank 38: 26029.29 + rank 39: 26029.31 + rank 40: 26028.50 + rank 41: 26028.49 + rank 42: 26028.51 + rank 43: 26028.51 + rank 44: 26028.48 + rank 45: 26028.50 + rank 46: 26028.48 + rank 47: 26028.48 + rank 48: 26029.06 + rank 49: 26028.95 + rank 50: 26028.95 + rank 51: 26029.09 + rank 52: 26028.89 + rank 53: 26029.16 + rank 54: 26029.01 + rank 55: 26029.02 + rank 56: 26028.82 + rank 57: 26028.87 + rank 58: 26028.85 + rank 59: 26028.87 + rank 60: 26028.89 + rank 61: 26028.84 + rank 62: 26028.82 + rank 63: 26028.83 + [2026-10-16 17:07:12] iteration 1961/ 2384 | consumed samples: 62752 | elapsed time per iteration (ms): 229058.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 73244.3 | valid tokens/s: 73244.3 | physical tokens/s: 74014.4 | loss tokens/s: 54810.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.231524E-01 | lm-global-supervised-tokens: 1.255483E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.620668E-01 | lm-token-buffer-mib: 2.356098E+02 | global_load_balancing_loss: 1.000231E+00 | loss scale: 1.0 | grad norm: 0.598 | params norm: 11153.392 | params rms: 0.060392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:54:51.623694 | finish at 2026-10-17 20:02:03 + [2026-10-16 17:10:59] iteration 1962/ 2384 | consumed samples: 62784 | elapsed time per iteration (ms): 227528.1 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 73736.7 | valid tokens/s: 73736.7 | physical tokens/s: 74843.7 | loss tokens/s: 58247.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.888353E-01 | lm-global-supervised-tokens: 1.325289E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.013067E-02 | lm-token-buffer-mib: 2.342834E+02 | global_load_balancing_loss: 9.995705E-01 | loss scale: 1.0 | grad norm: 0.256 | params norm: 11153.364 | params rms: 0.060392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:40:16.875109 | finish at 2026-10-17 19:51:16 + [2026-10-16 17:14:39] iteration 1963/ 2384 | consumed samples: 62816 | elapsed time per iteration (ms): 219764.8 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 76341.4 | valid tokens/s: 76341.4 | physical tokens/s: 77161.2 | loss tokens/s: 57700.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.622435E-01 | lm-global-supervised-tokens: 1.268062E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.558385E-02 | lm-token-buffer-mib: 2.336555E+02 | global_load_balancing_loss: 9.995968E-01 | loss scale: 1.0 | grad norm: 0.885 | params norm: 11153.333 | params rms: 0.060392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:42:00.976313 | finish at 2026-10-17 18:56:40 + [2026-10-16 17:18:25] iteration 1964/ 2384 | consumed samples: 62848 | elapsed time per iteration (ms): 225750.1 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 74317.4 | valid tokens/s: 74317.4 | physical tokens/s: 75819.3 | loss tokens/s: 57374.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.310892E-01 | lm-global-supervised-tokens: 1.295229E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.866677E-02 | lm-token-buffer-mib: 2.373873E+02 | global_load_balancing_loss: 9.987738E-01 | loss scale: 1.0 | grad norm: 0.247 | params norm: 11153.300 | params rms: 0.060391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:20:15.054073 | finish at 2026-10-17 19:38:40 + [2026-10-16 17:22:06] iteration 1965/ 2384 | consumed samples: 62880 | elapsed time per iteration (ms): 221932.8 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 75595.8 | valid tokens/s: 75595.8 | physical tokens/s: 76359.6 | loss tokens/s: 56733.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.581483E-01 | lm-global-supervised-tokens: 1.259094E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.245000E-02 | lm-token-buffer-mib: 2.357923E+02 | global_load_balancing_loss: 1.000985E+00 | loss scale: 1.0 | grad norm: 0.778 | params norm: 11153.266 | params rms: 0.060391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:49:49.836630 | finish at 2026-10-17 19:11:56 + [2026-10-16 17:25:48] iteration 1966/ 2384 | consumed samples: 62912 | elapsed time per iteration (ms): 221634.8 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 75697.5 | valid tokens/s: 75697.5 | physical tokens/s: 76297.3 | loss tokens/s: 54980.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.736749E-01 | lm-global-supervised-tokens: 1.218564E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.212390E-02 | lm-token-buffer-mib: 2.349360E+02 | global_load_balancing_loss: 1.002273E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11153.231 | params rms: 0.060391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:44:03.337114 | finish at 2026-10-17 19:09:51 + [2026-10-16 17:29:31] iteration 1967/ 2384 | consumed samples: 62944 | elapsed time per iteration (ms): 223102.4 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 75199.5 | valid tokens/s: 75199.5 | physical tokens/s: 75682.7 | loss tokens/s: 54744.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.826199E-01 | lm-global-supervised-tokens: 1.221357E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.954193E-02 | lm-token-buffer-mib: 2.344044E+02 | global_load_balancing_loss: 1.002149E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11153.195 | params rms: 0.060391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:50:33.701722 | finish at 2026-10-17 19:20:05 + [2026-10-16 17:33:14] iteration 1968/ 2384 | consumed samples: 62976 | elapsed time per iteration (ms): 223052.4 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.96% | FLOPs estimator: workload-aware | logical tokens/s: 75216.3 | valid tokens/s: 75216.3 | physical tokens/s: 76212.7 | loss tokens/s: 57863.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.286983E-01 | lm-global-supervised-tokens: 1.290669E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.509462E-02 | lm-token-buffer-mib: 2.352338E+02 | global_load_balancing_loss: 1.001059E+00 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11153.161 | params rms: 0.060391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:46:29.805389 | finish at 2026-10-17 19:19:44 + [2026-10-16 17:36:59] iteration 1969/ 2384 | consumed samples: 63008 | elapsed time per iteration (ms): 225028.9 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 74555.7 | valid tokens/s: 74555.7 | physical tokens/s: 75479.6 | loss tokens/s: 57746.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.435941E-01 | lm-global-supervised-tokens: 1.299453E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.312051E-02 | lm-token-buffer-mib: 2.353009E+02 | global_load_balancing_loss: 1.000605E+00 | loss scale: 1.0 | grad norm: 0.428 | params norm: 11153.129 | params rms: 0.060391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:56:26.990494 | finish at 2026-10-17 19:33:26 + [2026-10-16 17:41:06] iteration 1970/ 2384 | consumed samples: 63040 | elapsed time per iteration (ms): 246551.3 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.14% | FLOPs estimator: workload-aware | logical tokens/s: 68047.4 | valid tokens/s: 68047.4 | physical tokens/s: 68377.0 | loss tokens/s: 48897.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.802209E-01 | lm-global-supervised-tokens: 1.205569E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.796364E-02 | lm-token-buffer-mib: 2.328160E+02 | global_load_balancing_loss: 1.000587E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11153.096 | params rms: 0.060390 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:21:12.221243 | finish at 2026-10-17 22:02:18 + [2026-10-16 17:44:46] iteration 1971/ 2384 | consumed samples: 63072 | elapsed time per iteration (ms): 220422.8 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 76113.7 | valid tokens/s: 76113.7 | physical tokens/s: 76479.1 | loss tokens/s: 58211.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.372039E-01 | lm-global-supervised-tokens: 1.283120E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.568643E-02 | lm-token-buffer-mib: 2.336850E+02 | global_load_balancing_loss: 9.986445E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11153.060 | params rms: 0.060390 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:17:14.604118 | finish at 2026-10-17 19:02:01 + [2026-10-16 17:48:31] iteration 1972/ 2384 | consumed samples: 63104 | elapsed time per iteration (ms): 224371.2 | throughput per GPU (TFLOP/s/GPU): 17.6 | MFU 5.65% | FLOPs estimator: workload-aware | logical tokens/s: 74774.3 | valid tokens/s: 74774.3 | physical tokens/s: 75380.2 | loss tokens/s: 58589.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.426507E-01 | lm-global-supervised-tokens: 1.314585E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.860877E-02 | lm-token-buffer-mib: 2.341961E+02 | global_load_balancing_loss: 9.988567E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11153.023 | params rms: 0.060390 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:40:40.926085 | finish at 2026-10-17 19:29:12 + [2026-10-16 17:52:15] iteration 1973/ 2384 | consumed samples: 63136 | elapsed time per iteration (ms): 224694.4 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 74666.7 | valid tokens/s: 74666.7 | physical tokens/s: 75769.4 | loss tokens/s: 55948.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.161729E-01 | lm-global-supervised-tokens: 1.257133E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.310518E-02 | lm-token-buffer-mib: 2.340830E+02 | global_load_balancing_loss: 1.000805E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11152.989 | params rms: 0.060390 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:39:09.405877 | finish at 2026-10-17 19:31:25 + [2026-10-16 17:56:00] iteration 1974/ 2384 | consumed samples: 63168 | elapsed time per iteration (ms): 224437.8 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.36% | FLOPs estimator: workload-aware | logical tokens/s: 74752.1 | valid tokens/s: 74752.1 | physical tokens/s: 75158.8 | loss tokens/s: 54755.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.794890E-01 | lm-global-supervised-tokens: 1.228923E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.379695E-02 | lm-token-buffer-mib: 2.331137E+02 | global_load_balancing_loss: 1.000219E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11152.958 | params rms: 0.060390 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:33:39.518206 | finish at 2026-10-17 19:29:39 + [2026-10-16 17:59:42] iteration 1975/ 2384 | consumed samples: 63200 | elapsed time per iteration (ms): 222344.8 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75455.7 | valid tokens/s: 75455.7 | physical tokens/s: 76126.0 | loss tokens/s: 59250.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.119865E-01 | lm-global-supervised-tokens: 1.317404E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.001161E-02 | lm-token-buffer-mib: 2.329081E+02 | global_load_balancing_loss: 1.002158E+00 | loss scale: 1.0 | grad norm: 0.200 | params norm: 11152.926 | params rms: 0.060389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:15:39.007889 | finish at 2026-10-17 19:15:21 + [2026-10-16 18:03:25] iteration 1976/ 2384 | consumed samples: 63232 | elapsed time per iteration (ms): 223001.5 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 75233.6 | valid tokens/s: 75233.6 | physical tokens/s: 76048.1 | loss tokens/s: 57798.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.815625E-01 | lm-global-supervised-tokens: 1.288915E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.173237E-02 | lm-token-buffer-mib: 2.354229E+02 | global_load_balancing_loss: 1.000169E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11152.892 | params rms: 0.060389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:16:24.618668 | finish at 2026-10-17 19:19:50 + [2026-10-16 18:07:04] iteration 1977/ 2384 | consumed samples: 63264 | elapsed time per iteration (ms): 219126.9 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 76563.8 | valid tokens/s: 76563.8 | physical tokens/s: 77625.3 | loss tokens/s: 60896.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.297991E-01 | lm-global-supervised-tokens: 1.334409E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.394846E-02 | lm-token-buffer-mib: 2.336574E+02 | global_load_balancing_loss: 1.001352E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11152.862 | params rms: 0.060389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:46:24.661771 | finish at 2026-10-17 18:53:29 + [2026-10-16 18:10:44] iteration 1978/ 2384 | consumed samples: 63296 | elapsed time per iteration (ms): 219438.9 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 76454.9 | valid tokens/s: 76454.9 | physical tokens/s: 76957.6 | loss tokens/s: 58195.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.330319E-01 | lm-global-supervised-tokens: 1.277030E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.736134E-02 | lm-token-buffer-mib: 2.336792E+02 | global_load_balancing_loss: 1.001149E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11152.835 | params rms: 0.060389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:44:52.193978 | finish at 2026-10-17 18:55:36 + [2026-10-16 18:14:26] iteration 1979/ 2384 | consumed samples: 63328 | elapsed time per iteration (ms): 222491.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75406.1 | valid tokens/s: 75406.1 | physical tokens/s: 76509.1 | loss tokens/s: 58557.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.709150E-01 | lm-global-supervised-tokens: 1.302860E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.334299E-02 | lm-token-buffer-mib: 2.334068E+02 | global_load_balancing_loss: 1.000827E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11152.802 | params rms: 0.060389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:01:48.883846 | finish at 2026-10-17 19:16:15 + [2026-10-16 18:18:29] iteration 1980/ 2384 | consumed samples: 63360 | elapsed time per iteration (ms): 242381.3 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 69218.2 | valid tokens/s: 69218.2 | physical tokens/s: 69897.7 | loss tokens/s: 51083.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.561463E-01 | lm-global-supervised-tokens: 1.238157E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.883774E-02 | lm-token-buffer-mib: 2.354897E+02 | global_load_balancing_loss: 1.000204E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11152.767 | params rms: 0.060389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:12:02.035557 | finish at 2026-10-17 21:30:31 + [2026-10-16 18:22:10] iteration 1981/ 2384 | consumed samples: 63392 | elapsed time per iteration (ms): 221237.9 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 75833.3 | valid tokens/s: 75833.3 | physical tokens/s: 76157.1 | loss tokens/s: 54863.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.003700E-01 | lm-global-supervised-tokens: 1.213798E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.282500E-02 | lm-token-buffer-mib: 2.331562E+02 | global_load_balancing_loss: 1.000688E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11152.732 | params rms: 0.060388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:45:58.870345 | finish at 2026-10-17 19:08:09 + [2026-10-16 18:25:55] iteration 1982/ 2384 | consumed samples: 63424 | elapsed time per iteration (ms): 224777.7 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 74639.1 | valid tokens/s: 74639.1 | physical tokens/s: 75402.5 | loss tokens/s: 56243.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.912289E-01 | lm-global-supervised-tokens: 1.264226E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.480719E-02 | lm-token-buffer-mib: 2.359669E+02 | global_load_balancing_loss: 1.000673E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11152.699 | params rms: 0.060388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:06:00.652631 | finish at 2026-10-17 19:31:55 + [2026-10-16 18:29:36] iteration 1983/ 2384 | consumed samples: 63456 | elapsed time per iteration (ms): 221055.8 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 75895.7 | valid tokens/s: 75895.7 | physical tokens/s: 76504.4 | loss tokens/s: 58117.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.409704E-01 | lm-global-supervised-tokens: 1.284712E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.589386E-02 | lm-token-buffer-mib: 2.344433E+02 | global_load_balancing_loss: 1.000428E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11152.670 | params rms: 0.060388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:37:23.377601 | finish at 2026-10-17 19:06:59 + [2026-10-16 18:33:18] iteration 1984/ 2384 | consumed samples: 63488 | elapsed time per iteration (ms): 222617.9 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 75363.0 | valid tokens/s: 75363.0 | physical tokens/s: 76203.3 | loss tokens/s: 55877.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.040363E-01 | lm-global-supervised-tokens: 1.243931E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.270684E-02 | lm-token-buffer-mib: 2.332142E+02 | global_load_balancing_loss: 1.001482E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11152.642 | params rms: 0.060388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:44:07.167492 | finish at 2026-10-17 19:17:25 + [2026-10-16 18:37:04] iteration 1985/ 2384 | consumed samples: 63520 | elapsed time per iteration (ms): 225364.0 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 74444.9 | valid tokens/s: 74444.9 | physical tokens/s: 74678.7 | loss tokens/s: 55278.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.509659E-01 | lm-global-supervised-tokens: 1.245768E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.364035E-02 | lm-token-buffer-mib: 2.324381E+02 | global_load_balancing_loss: 1.000098E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11152.617 | params rms: 0.060388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:58:40.218339 | finish at 2026-10-17 19:35:44 + [2026-10-16 18:40:50] iteration 1986/ 2384 | consumed samples: 63552 | elapsed time per iteration (ms): 226212.3 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 74165.6 | valid tokens/s: 74165.6 | physical tokens/s: 74961.7 | loss tokens/s: 55377.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.595322E-01 | lm-global-supervised-tokens: 1.252701E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.520459E-02 | lm-token-buffer-mib: 2.348432E+02 | global_load_balancing_loss: 9.999644E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11152.592 | params rms: 0.060388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:00:32.492863 | finish at 2026-10-17 19:41:22 + [2026-10-16 18:44:35] iteration 1987/ 2384 | consumed samples: 63584 | elapsed time per iteration (ms): 224784.6 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 74636.7 | valid tokens/s: 74636.7 | physical tokens/s: 76222.2 | loss tokens/s: 54016.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.958098E-01 | lm-global-supervised-tokens: 1.214196E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.722876E-02 | lm-token-buffer-mib: 2.340011E+02 | global_load_balancing_loss: 1.000315E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11152.571 | params rms: 0.060388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:47:19.473808 | finish at 2026-10-17 19:31:54 + [2026-10-16 18:48:21] iteration 1988/ 2384 | consumed samples: 63616 | elapsed time per iteration (ms): 226234.5 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 74158.5 | valid tokens/s: 74158.5 | physical tokens/s: 75767.3 | loss tokens/s: 56158.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.130351E-01 | lm-global-supervised-tokens: 1.270493E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.244659E-02 | lm-token-buffer-mib: 2.363185E+02 | global_load_balancing_loss: 9.993272E-01 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11152.548 | params rms: 0.060387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:53:08.880572 | finish at 2026-10-17 19:41:30 + [2026-10-16 18:52:02] iteration 1989/ 2384 | consumed samples: 63648 | elapsed time per iteration (ms): 221434.0 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 75766.0 | valid tokens/s: 75766.0 | physical tokens/s: 76198.0 | loss tokens/s: 57145.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.230700E-01 | lm-global-supervised-tokens: 1.265405E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.766756E-02 | lm-token-buffer-mib: 2.336480E+02 | global_load_balancing_loss: 9.995440E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11152.525 | params rms: 0.060387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:17:46.440177 | finish at 2026-10-17 19:09:49 + [2026-10-16 18:56:07] iteration 1990/ 2384 | consumed samples: 63680 | elapsed time per iteration (ms): 244625.0 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.98% | FLOPs estimator: workload-aware | logical tokens/s: 68583.3 | valid tokens/s: 68583.3 | physical tokens/s: 69620.0 | loss tokens/s: 52226.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.887054E-01 | lm-global-supervised-tokens: 1.277602E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.379446E-02 | lm-token-buffer-mib: 2.365483E+02 | global_load_balancing_loss: 1.000820E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11152.504 | params rms: 0.060387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:46:22.251785 | finish at 2026-10-17 21:42:29 + [2026-10-16 18:59:48] iteration 1991/ 2384 | consumed samples: 63712 | elapsed time per iteration (ms): 220724.7 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 76009.5 | valid tokens/s: 76009.5 | physical tokens/s: 77077.0 | loss tokens/s: 63758.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.147884E-01 | lm-global-supervised-tokens: 1.407302E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.517647E-02 | lm-token-buffer-mib: 2.344137E+02 | global_load_balancing_loss: 1.000695E+00 | loss scale: 1.0 | grad norm: 0.196 | params norm: 11152.484 | params rms: 0.060387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:05:44.823019 | finish at 2026-10-17 19:05:32 + [2026-10-16 19:03:30] iteration 1992/ 2384 | consumed samples: 63744 | elapsed time per iteration (ms): 222121.7 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.58% | FLOPs estimator: workload-aware | logical tokens/s: 75531.6 | valid tokens/s: 75531.6 | physical tokens/s: 76661.1 | loss tokens/s: 56501.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.418422E-01 | lm-global-supervised-tokens: 1.255013E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.028666E-02 | lm-token-buffer-mib: 2.350262E+02 | global_load_balancing_loss: 1.000689E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11152.462 | params rms: 0.060387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:11:11.695484 | finish at 2026-10-17 19:14:41 + [2026-10-16 19:07:11] iteration 1993/ 2384 | consumed samples: 63776 | elapsed time per iteration (ms): 221526.0 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 75734.7 | valid tokens/s: 75734.7 | physical tokens/s: 76446.9 | loss tokens/s: 57604.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.695890E-01 | lm-global-supervised-tokens: 1.276084E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.304574E-02 | lm-token-buffer-mib: 2.334383E+02 | global_load_balancing_loss: 1.001720E+00 | loss scale: 1.0 | grad norm: 0.218 | params norm: 11152.440 | params rms: 0.060387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:03:36.664051 | finish at 2026-10-17 19:10:48 + [2026-10-16 19:10:51] iteration 1994/ 2384 | consumed samples: 63808 | elapsed time per iteration (ms): 219720.0 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.24% | FLOPs estimator: workload-aware | logical tokens/s: 76357.1 | valid tokens/s: 76357.1 | physical tokens/s: 76659.3 | loss tokens/s: 59510.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.870210E-01 | lm-global-supervised-tokens: 1.307562E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.509494E-02 | lm-token-buffer-mib: 2.330138E+02 | global_load_balancing_loss: 1.001748E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11152.416 | params rms: 0.060387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:48:10.810239 | finish at 2026-10-17 18:59:02 + [2026-10-16 19:14:36] iteration 1995/ 2384 | consumed samples: 63840 | elapsed time per iteration (ms): 224988.9 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 74568.9 | valid tokens/s: 74568.9 | physical tokens/s: 75936.8 | loss tokens/s: 59838.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.645096E-01 | lm-global-supervised-tokens: 1.346297E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.741372E-02 | lm-token-buffer-mib: 2.382797E+02 | global_load_balancing_loss: 1.000608E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11152.390 | params rms: 0.060387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:18:40.673639 | finish at 2026-10-17 19:33:17 + [2026-10-16 19:18:18] iteration 1996/ 2384 | consumed samples: 63872 | elapsed time per iteration (ms): 221536.5 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 75731.1 | valid tokens/s: 75731.1 | physical tokens/s: 76151.8 | loss tokens/s: 57081.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.838608E-01 | lm-global-supervised-tokens: 1.264573E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.142196E-02 | lm-token-buffer-mib: 2.338685E+02 | global_load_balancing_loss: 9.982662E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11152.366 | params rms: 0.060386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:52:36.168929 | finish at 2026-10-17 19:10:54 + [2026-10-16 19:22:01] iteration 1997/ 2384 | consumed samples: 63904 | elapsed time per iteration (ms): 223074.3 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 75209.0 | valid tokens/s: 75209.0 | physical tokens/s: 75942.1 | loss tokens/s: 55849.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.702866E-01 | lm-global-supervised-tokens: 1.245866E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.480974E-02 | lm-token-buffer-mib: 2.333261E+02 | global_load_balancing_loss: 9.995025E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11152.340 | params rms: 0.060386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:58:49.770728 | finish at 2026-10-17 19:20:50 + [2026-10-16 19:25:42] iteration 1998/ 2384 | consumed samples: 63936 | elapsed time per iteration (ms): 220901.4 | throughput per GPU (TFLOP/s/GPU): 26.0 | MFU 8.33% | FLOPs estimator: workload-aware | logical tokens/s: 75948.7 | valid tokens/s: 75948.7 | physical tokens/s: 76658.4 | loss tokens/s: 56033.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.461400E-01 | lm-global-supervised-tokens: 1.237787E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.653079E-02 | lm-token-buffer-mib: 2.344470E+02 | global_load_balancing_loss: 9.992684E-01 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11152.313 | params rms: 0.060386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:41:07.949177 | finish at 2026-10-17 19:06:49 + [2026-10-16 19:29:27] iteration 1999/ 2384 | consumed samples: 63968 | elapsed time per iteration (ms): 225952.2 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 74251.0 | valid tokens/s: 74251.0 | physical tokens/s: 74953.3 | loss tokens/s: 54759.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.242782E-01 | lm-global-supervised-tokens: 1.237296E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.117684E-02 | lm-token-buffer-mib: 2.342367E+02 | global_load_balancing_loss: 1.000865E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11152.284 | params rms: 0.060386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:09:51.615517 | finish at 2026-10-17 19:39:19 + [2026-10-16 19:33:33] iteration 2000/ 2384 | consumed samples: 64000 | elapsed time per iteration (ms): 245985.5 | throughput per GPU (TFLOP/s/GPU): 17.9 | MFU 5.73% | FLOPs estimator: workload-aware | logical tokens/s: 68204.1 | valid tokens/s: 68204.1 | physical tokens/s: 68816.6 | loss tokens/s: 50834.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.717943E-01 | lm-global-supervised-tokens: 1.250461E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.389248E-02 | lm-token-buffer-mib: 2.342782E+02 | global_load_balancing_loss: 9.997483E-01 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11152.254 | params rms: 0.060386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:14:18.417572 | finish at 2026-10-17 21:47:52 +times across ranks (ms): + save-checkpoint: + rank 0: 25574.64 + rank 1: 25574.65 + rank 2: 25574.62 + rank 3: 25574.72 + rank 4: 25574.62 + rank 5: 25574.67 + rank 6: 25574.71 + rank 7: 25574.61 + rank 8: 25573.92 + rank 9: 25573.89 + rank 10: 25573.89 + rank 11: 25573.94 + rank 12: 25573.87 + rank 13: 25573.98 + rank 14: 25573.91 + rank 15: 25573.83 + rank 16: 25573.93 + rank 17: 25573.92 + rank 18: 25573.93 + rank 19: 25573.92 + rank 20: 25573.91 + rank 21: 25573.91 + rank 22: 25573.92 + rank 23: 25573.91 + rank 24: 25575.47 + rank 25: 25575.50 + rank 26: 25575.47 + rank 27: 25575.49 + rank 28: 25575.35 + rank 29: 25575.87 + rank 30: 25575.46 + rank 31: 25575.47 + rank 32: 25572.12 + rank 33: 25572.07 + rank 34: 25572.18 + rank 35: 25572.14 + rank 36: 25572.04 + rank 37: 25572.15 + rank 38: 25572.00 + rank 39: 25572.00 + rank 40: 25574.01 + rank 41: 25573.96 + rank 42: 25573.96 + rank 43: 25573.98 + rank 44: 25573.94 + rank 45: 25573.94 + rank 46: 25573.93 + rank 47: 25573.95 + rank 48: 25573.51 + rank 49: 25573.48 + rank 50: 25573.55 + rank 51: 25573.58 + rank 52: 25573.46 + rank 53: 25573.56 + rank 54: 25573.47 + rank 55: 25573.52 + rank 56: 25574.04 + rank 57: 25574.14 + rank 58: 25574.14 + rank 59: 25574.09 + rank 60: 25574.09 + rank 61: 25573.90 + rank 62: 25574.05 + rank 63: 25574.06 + [2026-10-16 19:37:55] iteration 2001/ 2384 | consumed samples: 64032 | elapsed time per iteration (ms): 236188.0 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 71033.2 | valid tokens/s: 71033.2 | physical tokens/s: 71554.9 | loss tokens/s: 45252.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.857687E-01 | lm-global-supervised-tokens: 1.068818E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.205349E-01 | lm-token-buffer-mib: 2.352822E+02 | global_load_balancing_loss: 1.000858E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11152.225 | params rms: 0.060386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:07:40.010013 | finish at 2026-10-17 20:45:35 +times across ranks (ms): + forward-backward: + rank 0: 235591.45 + rank 1: 235587.08 + rank 2: 235587.72 + rank 3: 235589.25 + rank 4: 235595.11 + rank 5: 235587.97 + rank 6: 235590.11 + rank 7: 235590.42 + rank 8: 235994.78 + rank 9: 235991.91 + rank 10: 235989.59 + rank 11: 235992.88 + rank 12: 235997.64 + rank 13: 235993.83 + rank 14: 235991.28 + rank 15: 235997.58 + rank 16: 235261.91 + rank 17: 235264.58 + rank 18: 235258.67 + rank 19: 235265.56 + rank 20: 235262.97 + rank 21: 235261.56 + rank 22: 235257.55 + rank 23: 235264.92 + rank 24: 235711.14 + rank 25: 235705.70 + rank 26: 235703.58 + rank 27: 235704.53 + rank 28: 235708.83 + rank 29: 235708.39 + rank 30: 235706.23 + rank 31: 235711.50 + rank 32: 235550.09 + rank 33: 235545.48 + rank 34: 235545.03 + rank 35: 235545.89 + rank 36: 235553.59 + rank 37: 235545.59 + rank 38: 235549.36 + rank 39: 235550.67 + rank 40: 235985.50 + rank 41: 235979.50 + rank 42: 235977.17 + rank 43: 235980.36 + rank 44: 235982.28 + rank 45: 235976.34 + rank 46: 235983.23 + rank 47: 235986.69 + rank 48: 235262.98 + rank 49: 235264.11 + rank 50: 235256.81 + rank 51: 235264.03 + rank 52: 235259.00 + rank 53: 235258.47 + rank 54: 235260.59 + rank 55: 235266.62 + rank 56: 235713.44 + rank 57: 235707.52 + rank 58: 235703.33 + rank 59: 235704.80 + rank 60: 235703.58 + rank 61: 235703.19 + rank 62: 235710.94 + rank 63: 235714.03 + forward-compute: + rank 0: 130949.71 + rank 1: 130940.59 + rank 2: 130930.24 + rank 3: 130924.23 + rank 4: 130936.09 + rank 5: 130945.93 + rank 6: 130944.07 + rank 7: 130951.23 + rank 8: 131360.72 + rank 9: 131361.02 + rank 10: 131351.91 + rank 11: 131367.17 + rank 12: 131390.77 + rank 13: 131375.28 + rank 14: 131356.23 + rank 15: 131370.81 + rank 16: 130602.50 + rank 17: 130600.27 + rank 18: 130594.08 + rank 19: 130602.88 + rank 20: 130599.62 + rank 21: 130609.51 + rank 22: 130608.38 + rank 23: 130613.18 + rank 24: 131347.30 + rank 25: 131349.34 + rank 26: 131749.78 + rank 27: 131576.00 + rank 28: 131358.27 + rank 29: 131599.56 + rank 30: 131396.31 + rank 31: 131684.66 + rank 32: 74356.23 + rank 33: 74343.27 + rank 34: 74347.02 + rank 35: 74354.91 + rank 36: 74369.98 + rank 37: 74390.70 + rank 38: 74358.26 + rank 39: 74369.74 + rank 40: 74831.92 + rank 41: 74792.77 + rank 42: 74797.05 + rank 43: 74810.97 + rank 44: 74796.94 + rank 45: 74800.75 + rank 46: 74767.56 + rank 47: 74817.92 + rank 48: 74049.98 + rank 49: 74053.02 + rank 50: 74059.63 + rank 51: 74067.02 + rank 52: 74093.05 + rank 53: 74084.70 + rank 54: 74040.34 + rank 55: 74070.24 + rank 56: 74503.58 + rank 57: 74484.06 + rank 58: 74475.55 + rank 59: 74470.29 + rank 60: 74490.30 + rank 61: 74518.98 + rank 62: 74474.37 + rank 63: 74512.03 + backward-compute: + rank 0: 104594.59 + rank 1: 104584.73 + rank 2: 104606.65 + rank 3: 104615.88 + rank 4: 104600.46 + rank 5: 104593.02 + rank 6: 104590.71 + rank 7: 104592.87 + rank 8: 104585.55 + rank 9: 104578.73 + rank 10: 104584.97 + rank 11: 104574.95 + rank 12: 104555.27 + rank 13: 104566.50 + rank 14: 104582.08 + rank 15: 104576.85 + rank 16: 104618.73 + rank 17: 104614.53 + rank 18: 104615.05 + rank 19: 104610.50 + rank 20: 104612.98 + rank 21: 104604.57 + rank 22: 104600.35 + rank 23: 104606.74 + rank 24: 104305.66 + rank 25: 104298.05 + rank 26: 103890.01 + rank 27: 104068.66 + rank 28: 104281.76 + rank 29: 104040.59 + rank 30: 104251.52 + rank 31: 103962.24 + rank 32: 160985.22 + rank 33: 160992.08 + rank 34: 160988.55 + rank 35: 160980.34 + rank 36: 160974.12 + rank 37: 160944.69 + rank 38: 160980.38 + rank 39: 160971.27 + rank 40: 161118.14 + rank 41: 161150.77 + rank 42: 161145.31 + rank 43: 161134.00 + rank 44: 161150.89 + rank 45: 161140.92 + rank 46: 161179.50 + rank 47: 161135.03 + rank 48: 161180.14 + rank 49: 161176.20 + rank 50: 161159.59 + rank 51: 161160.00 + rank 52: 161133.91 + rank 53: 161137.12 + rank 54: 161185.28 + rank 55: 161161.38 + rank 56: 161174.67 + rank 57: 161183.80 + rank 58: 161191.70 + rank 59: 161195.59 + rank 60: 161177.48 + rank 61: 161146.77 + rank 62: 161199.78 + rank 63: 161164.52 + batch-generator: + rank 0: 34.55 + rank 1: 35.71 + rank 2: 36.50 + rank 3: 35.71 + rank 4: 37.90 + rank 5: 36.29 + rank 6: 35.89 + rank 7: 33.47 + rank 8: 42.11 + rank 9: 47.45 + rank 10: 43.05 + rank 11: 47.88 + rank 12: 43.53 + rank 13: 43.76 + rank 14: 44.56 + rank 15: 42.28 + rank 16: 32.23 + rank 17: 35.17 + rank 18: 37.30 + rank 19: 36.66 + rank 20: 36.09 + rank 21: 36.38 + rank 22: 37.37 + rank 23: 32.56 + rank 24: 45.03 + rank 25: 46.35 + rank 26: 50.38 + rank 27: 49.45 + rank 28: 47.11 + rank 29: 47.68 + rank 30: 45.42 + rank 31: 44.47 + rank 32: 46.02 + rank 33: 51.96 + rank 34: 48.82 + rank 35: 50.61 + rank 36: 46.78 + rank 37: 47.32 + rank 38: 46.36 + rank 39: 41.67 + rank 40: 35.02 + rank 41: 37.74 + rank 42: 36.56 + rank 43: 38.37 + rank 44: 37.44 + rank 45: 36.31 + rank 46: 36.94 + rank 47: 32.92 + rank 48: 40.59 + rank 49: 42.98 + rank 50: 44.76 + rank 51: 45.58 + rank 52: 33.71 + rank 53: 42.58 + rank 54: 36.14 + rank 55: 38.50 + rank 56: 32.82 + rank 57: 38.94 + rank 58: 39.34 + rank 59: 39.57 + rank 60: 42.04 + rank 61: 37.25 + rank 62: 38.28 + rank 63: 33.95 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.04 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.04 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.04 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.04 + rank 34: 0.03 + rank 35: 0.04 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.03 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.86 + rank 1: 0.86 + rank 2: 1.03 + rank 3: 0.96 + rank 4: 1.11 + rank 5: 0.86 + rank 6: 0.91 + rank 7: 1.05 + rank 8: 0.96 + rank 9: 0.91 + rank 10: 1.08 + rank 11: 0.87 + rank 12: 0.91 + rank 13: 1.23 + rank 14: 0.86 + rank 15: 1.23 + rank 16: 1.10 + rank 17: 0.86 + rank 18: 0.89 + rank 19: 0.86 + rank 20: 0.85 + rank 21: 1.06 + rank 22: 0.91 + rank 23: 1.09 + rank 24: 1.24 + rank 25: 1.35 + rank 26: 1.28 + rank 27: 1.62 + rank 28: 1.68 + rank 29: 1.41 + rank 30: 1.55 + rank 31: 1.29 + rank 32: 1.20 + rank 33: 1.34 + rank 34: 1.48 + rank 35: 1.36 + rank 36: 1.18 + rank 37: 1.31 + rank 38: 1.36 + rank 39: 1.18 + rank 40: 1.11 + rank 41: 0.91 + rank 42: 0.84 + rank 43: 1.02 + rank 44: 1.14 + rank 45: 0.87 + rank 46: 1.87 + rank 47: 0.84 + rank 48: 1.23 + rank 49: 1.26 + rank 50: 1.06 + rank 51: 0.99 + rank 52: 1.12 + rank 53: 1.07 + rank 54: 0.90 + rank 55: 1.08 + rank 56: 1.21 + rank 57: 0.86 + rank 58: 1.15 + rank 59: 1.30 + rank 60: 0.82 + rank 61: 0.89 + rank 62: 1.45 + rank 63: 1.27 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.03 + rank 34: 0.02 + rank 35: 0.03 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.36 + rank 1: 1.87 + rank 2: 1.80 + rank 3: 2.16 + rank 4: 1.69 + rank 5: 1.94 + rank 6: 1.90 + rank 7: 1.88 + rank 8: 1.92 + rank 9: 2.30 + rank 10: 1.78 + rank 11: 1.85 + rank 12: 1.89 + rank 13: 1.82 + rank 14: 1.98 + rank 15: 2.17 + rank 16: 3.07 + rank 17: 2.57 + rank 18: 2.30 + rank 19: 2.42 + rank 20: 2.62 + rank 21: 2.31 + rank 22: 2.76 + rank 23: 2.41 + rank 24: 2.07 + rank 25: 2.23 + rank 26: 2.16 + rank 27: 2.23 + rank 28: 2.33 + rank 29: 2.43 + rank 30: 2.22 + rank 31: 2.23 + rank 32: 2.10 + rank 33: 1.97 + rank 34: 2.02 + rank 35: 2.55 + rank 36: 2.05 + rank 37: 2.09 + rank 38: 2.55 + rank 39: 1.94 + rank 40: 1.77 + rank 41: 2.08 + rank 42: 1.92 + rank 43: 1.95 + rank 44: 1.90 + rank 45: 2.13 + rank 46: 1.77 + rank 47: 1.77 + rank 48: 1.95 + rank 49: 1.85 + rank 50: 2.50 + rank 51: 1.83 + rank 52: 2.06 + rank 53: 1.83 + rank 54: 1.82 + rank 55: 1.88 + rank 56: 1.80 + rank 57: 2.46 + rank 58: 1.81 + rank 59: 1.82 + rank 60: 1.75 + rank 61: 1.78 + rank 62: 1.86 + rank 63: 1.75 + optimizer-inner-step: + rank 0: 44.80 + rank 1: 45.46 + rank 2: 44.52 + rank 3: 44.59 + rank 4: 44.29 + rank 5: 44.51 + rank 6: 44.83 + rank 7: 45.66 + rank 8: 45.56 + rank 9: 45.55 + rank 10: 45.36 + rank 11: 45.56 + rank 12: 45.58 + rank 13: 45.72 + rank 14: 45.47 + rank 15: 45.29 + rank 16: 45.85 + rank 17: 45.48 + rank 18: 45.54 + rank 19: 45.51 + rank 20: 45.45 + rank 21: 45.41 + rank 22: 46.00 + rank 23: 45.68 + rank 24: 45.23 + rank 25: 45.42 + rank 26: 45.36 + rank 27: 45.46 + rank 28: 46.68 + rank 29: 45.21 + rank 30: 45.42 + rank 31: 45.56 + rank 32: 45.60 + rank 33: 45.35 + rank 34: 45.53 + rank 35: 55.79 + rank 36: 47.83 + rank 37: 48.06 + rank 38: 48.35 + rank 39: 45.39 + rank 40: 45.47 + rank 41: 45.62 + rank 42: 45.61 + rank 43: 45.69 + rank 44: 45.29 + rank 45: 45.86 + rank 46: 45.52 + rank 47: 45.27 + rank 48: 45.48 + rank 49: 45.34 + rank 50: 55.35 + rank 51: 45.22 + rank 52: 45.51 + rank 53: 45.42 + rank 54: 45.61 + rank 55: 45.54 + rank 56: 45.20 + rank 57: 47.57 + rank 58: 45.49 + rank 59: 45.29 + rank 60: 45.58 + rank 61: 45.27 + rank 62: 45.39 + rank 63: 45.33 + optimizer-copy-main-to-model-params: + rank 0: 4.58 + rank 1: 3.96 + rank 2: 3.82 + rank 3: 4.47 + rank 4: 3.73 + rank 5: 4.04 + rank 6: 4.00 + rank 7: 4.00 + rank 8: 3.96 + rank 9: 4.49 + rank 10: 3.81 + rank 11: 3.86 + rank 12: 3.96 + rank 13: 3.86 + rank 14: 4.05 + rank 15: 4.29 + rank 16: 4.52 + rank 17: 4.02 + rank 18: 4.14 + rank 19: 4.08 + rank 20: 4.08 + rank 21: 4.23 + rank 22: 4.69 + rank 23: 4.24 + rank 24: 3.91 + rank 25: 4.21 + rank 26: 4.01 + rank 27: 4.07 + rank 28: 4.37 + rank 29: 4.39 + rank 30: 4.33 + rank 31: 4.02 + rank 32: 4.36 + rank 33: 4.04 + rank 34: 4.14 + rank 35: 4.77 + rank 36: 4.44 + rank 37: 4.41 + rank 38: 4.59 + rank 39: 4.12 + rank 40: 3.82 + rank 41: 4.00 + rank 42: 4.04 + rank 43: 4.06 + rank 44: 4.07 + rank 45: 4.31 + rank 46: 3.79 + rank 47: 3.77 + rank 48: 4.07 + rank 49: 3.94 + rank 50: 4.99 + rank 51: 4.08 + rank 52: 4.38 + rank 53: 3.93 + rank 54: 3.94 + rank 55: 4.00 + rank 56: 3.79 + rank 57: 4.70 + rank 58: 3.83 + rank 59: 3.71 + rank 60: 3.81 + rank 61: 3.73 + rank 62: 3.92 + rank 63: 3.74 + optimizer: + rank 0: 59.58 + rank 1: 58.55 + rank 2: 58.09 + rank 3: 59.18 + rank 4: 57.92 + rank 5: 58.53 + rank 6: 58.88 + rank 7: 58.80 + rank 8: 58.58 + rank 9: 59.30 + rank 10: 58.21 + rank 11: 58.43 + rank 12: 58.59 + rank 13: 58.59 + rank 14: 58.64 + rank 15: 58.78 + rank 16: 59.88 + rank 17: 58.90 + rank 18: 59.10 + rank 19: 58.94 + rank 20: 58.89 + rank 21: 59.08 + rank 22: 60.29 + rank 23: 59.35 + rank 24: 58.23 + rank 25: 58.89 + rank 26: 58.57 + rank 27: 58.76 + rank 28: 60.36 + rank 29: 58.96 + rank 30: 58.99 + rank 31: 58.70 + rank 32: 59.03 + rank 33: 58.39 + rank 34: 58.68 + rank 35: 69.78 + rank 36: 61.35 + rank 37: 61.54 + rank 38: 62.14 + rank 39: 58.49 + rank 40: 58.30 + rank 41: 58.68 + rank 42: 58.73 + rank 43: 58.85 + rank 44: 58.43 + rank 45: 59.37 + rank 46: 58.29 + rank 47: 58.00 + rank 48: 58.68 + rank 49: 58.33 + rank 50: 69.74 + rank 51: 58.32 + rank 52: 59.04 + rank 53: 58.35 + rank 54: 58.54 + rank 55: 58.52 + rank 56: 57.99 + rank 57: 61.69 + rank 58: 58.36 + rank 59: 57.99 + rank 60: 58.39 + rank 61: 57.99 + rank 62: 58.38 + rank 63: 58.07 + interval-time: + rank 0: 0.85 + rank 1: 0.85 + rank 2: 0.89 + rank 3: 0.78 + rank 4: 0.82 + rank 5: 0.86 + rank 6: 0.73 + rank 7: 0.73 + rank 8: 0.77 + rank 9: 0.88 + rank 10: 0.86 + rank 11: 0.85 + rank 12: 0.86 + rank 13: 0.74 + rank 14: 0.86 + rank 15: 0.84 + rank 16: 0.84 + rank 17: 0.84 + rank 18: 0.76 + rank 19: 0.86 + rank 20: 0.74 + rank 21: 0.81 + rank 22: 0.76 + rank 23: 0.83 + rank 24: 1.04 + rank 25: 1.04 + rank 26: 0.99 + rank 27: 0.96 + rank 28: 1.28 + rank 29: 1.29 + rank 30: 1.11 + rank 31: 1.30 + rank 32: 0.83 + rank 33: 0.86 + rank 34: 0.94 + rank 35: 0.95 + rank 36: 0.92 + rank 37: 1.03 + rank 38: 0.92 + rank 39: 0.91 + rank 40: 0.85 + rank 41: 0.76 + rank 42: 0.87 + rank 43: 0.76 + rank 44: 0.74 + rank 45: 0.84 + rank 46: 0.85 + rank 47: 0.82 + rank 48: 0.83 + rank 49: 0.97 + rank 50: 0.85 + rank 51: 0.91 + rank 52: 0.80 + rank 53: 0.85 + rank 54: 0.80 + rank 55: 0.81 + rank 56: 0.86 + rank 57: 0.84 + rank 58: 0.85 + rank 59: 0.85 + rank 60: 0.76 + rank 61: 0.86 + rank 62: 0.75 + rank 63: 2.94 + [2026-10-16 19:41:45] iteration 2002/ 2384 | consumed samples: 64064 | elapsed time per iteration (ms): 230015.7 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.65% | FLOPs estimator: workload-aware | logical tokens/s: 72939.3 | valid tokens/s: 72939.3 | physical tokens/s: 73345.7 | loss tokens/s: 47929.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.710937E-01 | lm-global-supervised-tokens: 1.102456E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.114318E-02 | lm-token-buffer-mib: 2.336087E+02 | global_load_balancing_loss: 1.000528E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11152.196 | params rms: 0.060386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:24:25.982047 | finish at 2026-10-17 20:06:11 +times across ranks (ms): + forward-backward: + rank 0: 224080.45 + rank 1: 224157.84 + rank 2: 224159.25 + rank 3: 224159.67 + rank 4: 224160.47 + rank 5: 224159.30 + rank 6: 224155.17 + rank 7: 224159.91 + rank 8: 224127.09 + rank 9: 224127.94 + rank 10: 224127.78 + rank 11: 224128.36 + rank 12: 224128.30 + rank 13: 224128.98 + rank 14: 224126.70 + rank 15: 224126.91 + rank 16: 224151.66 + rank 17: 224149.52 + rank 18: 224150.91 + rank 19: 224155.88 + rank 20: 224156.45 + rank 21: 224149.28 + rank 22: 224144.02 + rank 23: 224152.06 + rank 24: 224145.89 + rank 25: 224147.75 + rank 26: 224143.78 + rank 27: 224148.73 + rank 28: 224150.31 + rank 29: 224148.69 + rank 30: 224145.05 + rank 31: 224143.23 + rank 32: 224049.47 + rank 33: 224125.25 + rank 34: 224126.64 + rank 35: 224125.86 + rank 36: 224127.94 + rank 37: 224125.64 + rank 38: 224124.44 + rank 39: 224129.22 + rank 40: 224110.44 + rank 41: 224111.12 + rank 42: 224109.33 + rank 43: 224109.36 + rank 44: 224108.70 + rank 45: 224107.33 + rank 46: 224112.56 + rank 47: 224112.38 + rank 48: 224154.28 + rank 49: 224150.98 + rank 50: 224150.14 + rank 51: 224155.81 + rank 52: 224153.83 + rank 53: 224147.59 + rank 54: 224147.98 + rank 55: 224155.98 + rank 56: 224148.89 + rank 57: 224148.58 + rank 58: 224143.48 + rank 59: 224147.44 + rank 60: 224147.30 + rank 61: 224144.91 + rank 62: 224150.08 + rank 63: 224147.98 + forward-compute: + rank 0: 129681.28 + rank 1: 129761.77 + rank 2: 129760.95 + rank 3: 129762.64 + rank 4: 129767.59 + rank 5: 129765.45 + rank 6: 129765.07 + rank 7: 129768.66 + rank 8: 129797.91 + rank 9: 129791.25 + rank 10: 129799.32 + rank 11: 129816.66 + rank 12: 129804.86 + rank 13: 129789.95 + rank 14: 129793.68 + rank 15: 129794.47 + rank 16: 129776.74 + rank 17: 129769.16 + rank 18: 129762.78 + rank 19: 129770.45 + rank 20: 129769.88 + rank 21: 129761.34 + rank 22: 129762.17 + rank 23: 129770.61 + rank 24: 129766.20 + rank 25: 129763.97 + rank 26: 129756.16 + rank 27: 129774.87 + rank 28: 129781.79 + rank 29: 129779.73 + rank 30: 129779.99 + rank 31: 129776.07 + rank 32: 61349.07 + rank 33: 61414.96 + rank 34: 61413.30 + rank 35: 61427.74 + rank 36: 61444.02 + rank 37: 61447.02 + rank 38: 61446.49 + rank 39: 61446.27 + rank 40: 61438.23 + rank 41: 61404.32 + rank 42: 61429.64 + rank 43: 61416.92 + rank 44: 61422.59 + rank 45: 61416.75 + rank 46: 61396.64 + rank 47: 61430.07 + rank 48: 61430.19 + rank 49: 61442.70 + rank 50: 61417.38 + rank 51: 61422.38 + rank 52: 61474.73 + rank 53: 61466.71 + rank 54: 61415.66 + rank 55: 61436.83 + rank 56: 61435.36 + rank 57: 61424.67 + rank 58: 61406.72 + rank 59: 61425.70 + rank 60: 61437.93 + rank 61: 61454.15 + rank 62: 61409.29 + rank 63: 61452.98 + backward-compute: + rank 0: 94349.83 + rank 1: 94341.36 + rank 2: 94345.54 + rank 3: 94343.70 + rank 4: 94341.28 + rank 5: 94339.39 + rank 6: 94336.35 + rank 7: 94342.94 + rank 8: 94273.28 + rank 9: 94282.86 + rank 10: 94275.30 + rank 11: 94259.28 + rank 12: 94269.61 + rank 13: 94284.32 + rank 14: 94279.37 + rank 15: 94281.18 + rank 16: 94327.01 + rank 17: 94325.02 + rank 18: 94336.78 + rank 19: 94330.95 + rank 20: 94333.30 + rank 21: 94334.55 + rank 22: 94325.57 + rank 23: 94334.94 + rank 24: 94324.26 + rank 25: 94326.38 + rank 26: 94325.39 + rank 27: 94309.48 + rank 28: 94302.31 + rank 29: 94293.95 + rank 30: 94308.22 + rank 31: 94307.06 + rank 32: 162662.58 + rank 33: 162671.25 + rank 34: 162674.70 + rank 35: 162658.72 + rank 36: 162646.03 + rank 37: 162638.95 + rank 38: 162640.09 + rank 39: 162646.45 + rank 40: 162638.08 + rank 41: 162671.72 + rank 42: 162647.38 + rank 43: 162658.55 + rank 44: 162651.70 + rank 45: 162656.48 + rank 46: 162680.44 + rank 47: 162648.48 + rank 48: 162690.00 + rank 49: 162672.86 + rank 50: 162696.98 + rank 51: 162697.75 + rank 52: 162645.19 + rank 53: 162638.41 + rank 54: 162696.59 + rank 55: 162685.39 + rank 56: 162681.14 + rank 57: 162688.73 + rank 58: 162700.44 + rank 59: 162686.03 + rank 60: 162674.28 + rank 61: 162654.47 + rank 62: 162706.25 + rank 63: 162660.88 + batch-generator: + rank 0: 35.70 + rank 1: 36.20 + rank 2: 36.14 + rank 3: 37.40 + rank 4: 36.83 + rank 5: 38.94 + rank 6: 37.84 + rank 7: 33.17 + rank 8: 61.64 + rank 9: 62.18 + rank 10: 61.93 + rank 11: 67.85 + rank 12: 68.59 + rank 13: 60.16 + rank 14: 63.96 + rank 15: 58.89 + rank 16: 32.64 + rank 17: 35.53 + rank 18: 35.54 + rank 19: 36.60 + rank 20: 35.81 + rank 21: 35.43 + rank 22: 36.27 + rank 23: 31.13 + rank 24: 48.34 + rank 25: 49.36 + rank 26: 48.98 + rank 27: 48.40 + rank 28: 47.04 + rank 29: 51.17 + rank 30: 47.50 + rank 31: 44.62 + rank 32: 43.98 + rank 33: 48.88 + rank 34: 49.09 + rank 35: 52.38 + rank 36: 48.00 + rank 37: 50.69 + rank 38: 42.20 + rank 39: 38.63 + rank 40: 32.90 + rank 41: 36.63 + rank 42: 36.24 + rank 43: 36.74 + rank 44: 35.45 + rank 45: 34.21 + rank 46: 36.02 + rank 47: 29.99 + rank 48: 37.15 + rank 49: 40.08 + rank 50: 45.60 + rank 51: 44.04 + rank 52: 36.88 + rank 53: 48.17 + rank 54: 38.59 + rank 55: 36.42 + rank 56: 31.59 + rank 57: 38.66 + rank 58: 40.57 + rank 59: 36.88 + rank 60: 36.70 + rank 61: 37.57 + rank 62: 37.06 + rank 63: 30.44 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.03 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.82 + rank 1: 1.11 + rank 2: 1.03 + rank 3: 0.89 + rank 4: 1.10 + rank 5: 1.10 + rank 6: 0.86 + rank 7: 1.02 + rank 8: 0.89 + rank 9: 1.09 + rank 10: 1.03 + rank 11: 0.83 + rank 12: 0.85 + rank 13: 0.82 + rank 14: 1.07 + rank 15: 1.17 + rank 16: 0.81 + rank 17: 0.94 + rank 18: 1.14 + rank 19: 0.86 + rank 20: 1.21 + rank 21: 0.79 + rank 22: 1.19 + rank 23: 0.87 + rank 24: 1.56 + rank 25: 1.48 + rank 26: 1.36 + rank 27: 1.27 + rank 28: 1.19 + rank 29: 1.11 + rank 30: 1.17 + rank 31: 1.26 + rank 32: 1.27 + rank 33: 1.56 + rank 34: 1.68 + rank 35: 1.50 + rank 36: 1.37 + rank 37: 1.33 + rank 38: 1.52 + rank 39: 1.35 + rank 40: 0.85 + rank 41: 0.89 + rank 42: 0.85 + rank 43: 0.93 + rank 44: 1.12 + rank 45: 0.93 + rank 46: 0.94 + rank 47: 1.07 + rank 48: 0.95 + rank 49: 1.15 + rank 50: 1.09 + rank 51: 1.21 + rank 52: 1.43 + rank 53: 1.32 + rank 54: 0.90 + rank 55: 0.85 + rank 56: 0.94 + rank 57: 0.83 + rank 58: 1.14 + rank 59: 1.35 + rank 60: 1.08 + rank 61: 1.12 + rank 62: 1.00 + rank 63: 0.85 + params-all-gather: + rank 0: 0.02 + rank 1: 0.03 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.03 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.04 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.03 + rank 35: 0.02 + rank 36: 0.03 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.03 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.22 + rank 1: 1.82 + rank 2: 1.77 + rank 3: 2.17 + rank 4: 1.76 + rank 5: 1.99 + rank 6: 1.98 + rank 7: 1.92 + rank 8: 1.82 + rank 9: 2.16 + rank 10: 1.77 + rank 11: 1.78 + rank 12: 1.86 + rank 13: 1.77 + rank 14: 1.91 + rank 15: 2.15 + rank 16: 2.47 + rank 17: 2.22 + rank 18: 2.25 + rank 19: 2.22 + rank 20: 2.23 + rank 21: 2.27 + rank 22: 2.62 + rank 23: 2.34 + rank 24: 1.95 + rank 25: 2.02 + rank 26: 1.88 + rank 27: 2.23 + rank 28: 2.29 + rank 29: 2.51 + rank 30: 2.09 + rank 31: 2.22 + rank 32: 2.17 + rank 33: 1.92 + rank 34: 2.05 + rank 35: 2.34 + rank 36: 2.10 + rank 37: 2.04 + rank 38: 2.26 + rank 39: 1.94 + rank 40: 1.75 + rank 41: 1.90 + rank 42: 1.86 + rank 43: 1.94 + rank 44: 1.85 + rank 45: 2.11 + rank 46: 1.76 + rank 47: 1.70 + rank 48: 1.97 + rank 49: 1.85 + rank 50: 2.55 + rank 51: 1.82 + rank 52: 2.00 + rank 53: 1.94 + rank 54: 1.81 + rank 55: 1.80 + rank 56: 1.80 + rank 57: 2.40 + rank 58: 1.86 + rank 59: 1.73 + rank 60: 1.78 + rank 61: 1.70 + rank 62: 1.84 + rank 63: 1.82 + optimizer-inner-step: + rank 0: 46.02 + rank 1: 45.91 + rank 2: 45.68 + rank 3: 45.86 + rank 4: 45.61 + rank 5: 45.80 + rank 6: 46.13 + rank 7: 46.09 + rank 8: 38.44 + rank 9: 45.85 + rank 10: 45.72 + rank 11: 45.86 + rank 12: 45.83 + rank 13: 46.04 + rank 14: 45.81 + rank 15: 45.69 + rank 16: 46.22 + rank 17: 45.86 + rank 18: 45.99 + rank 19: 45.91 + rank 20: 45.84 + rank 21: 45.74 + rank 22: 46.39 + rank 23: 45.98 + rank 24: 45.66 + rank 25: 45.93 + rank 26: 45.84 + rank 27: 46.11 + rank 28: 46.33 + rank 29: 45.61 + rank 30: 45.78 + rank 31: 45.99 + rank 32: 45.86 + rank 33: 45.74 + rank 34: 45.80 + rank 35: 48.89 + rank 36: 45.99 + rank 37: 45.83 + rank 38: 45.74 + rank 39: 45.71 + rank 40: 45.71 + rank 41: 45.83 + rank 42: 45.93 + rank 43: 45.99 + rank 44: 45.58 + rank 45: 46.27 + rank 46: 45.85 + rank 47: 45.51 + rank 48: 45.80 + rank 49: 45.72 + rank 50: 57.81 + rank 51: 45.58 + rank 52: 45.91 + rank 53: 45.69 + rank 54: 45.97 + rank 55: 45.86 + rank 56: 45.56 + rank 57: 46.85 + rank 58: 45.85 + rank 59: 45.63 + rank 60: 45.92 + rank 61: 45.61 + rank 62: 45.71 + rank 63: 45.64 + optimizer-copy-main-to-model-params: + rank 0: 4.52 + rank 1: 3.89 + rank 2: 3.81 + rank 3: 4.45 + rank 4: 3.76 + rank 5: 4.11 + rank 6: 4.11 + rank 7: 4.06 + rank 8: 3.88 + rank 9: 4.44 + rank 10: 3.84 + rank 11: 3.83 + rank 12: 3.95 + rank 13: 3.83 + rank 14: 3.99 + rank 15: 4.25 + rank 16: 4.45 + rank 17: 3.94 + rank 18: 4.05 + rank 19: 4.03 + rank 20: 4.05 + rank 21: 4.21 + rank 22: 4.64 + rank 23: 4.21 + rank 24: 3.89 + rank 25: 4.22 + rank 26: 3.97 + rank 27: 4.56 + rank 28: 4.40 + rank 29: 4.41 + rank 30: 4.33 + rank 31: 4.08 + rank 32: 4.18 + rank 33: 3.89 + rank 34: 4.01 + rank 35: 4.53 + rank 36: 4.24 + rank 37: 4.11 + rank 38: 4.48 + rank 39: 3.88 + rank 40: 3.78 + rank 41: 4.05 + rank 42: 3.93 + rank 43: 3.99 + rank 44: 3.95 + rank 45: 4.22 + rank 46: 3.72 + rank 47: 3.67 + rank 48: 4.07 + rank 49: 3.90 + rank 50: 4.95 + rank 51: 4.01 + rank 52: 4.20 + rank 53: 3.92 + rank 54: 3.93 + rank 55: 4.00 + rank 56: 3.72 + rank 57: 4.73 + rank 58: 4.01 + rank 59: 3.69 + rank 60: 3.77 + rank 61: 3.71 + rank 62: 3.89 + rank 63: 3.75 + optimizer: + rank 0: 60.01 + rank 1: 59.04 + rank 2: 58.65 + rank 3: 59.73 + rank 4: 58.53 + rank 5: 59.15 + rank 6: 59.54 + rank 7: 59.44 + rank 8: 58.97 + rank 9: 59.72 + rank 10: 58.79 + rank 11: 58.89 + rank 12: 59.02 + rank 13: 59.07 + rank 14: 59.05 + rank 15: 59.30 + rank 16: 60.30 + rank 17: 59.29 + rank 18: 59.51 + rank 19: 59.40 + rank 20: 59.38 + rank 21: 59.53 + rank 22: 60.72 + rank 23: 59.77 + rank 24: 58.77 + rank 25: 59.44 + rank 26: 59.02 + rank 27: 59.91 + rank 28: 60.10 + rank 29: 59.40 + rank 30: 59.46 + rank 31: 59.31 + rank 32: 59.29 + rank 33: 58.78 + rank 34: 58.99 + rank 35: 62.81 + rank 36: 59.50 + rank 37: 59.17 + rank 38: 59.59 + rank 39: 58.76 + rank 40: 58.73 + rank 41: 59.21 + rank 42: 59.12 + rank 43: 59.26 + rank 44: 58.82 + rank 45: 59.89 + rank 46: 58.75 + rank 47: 58.37 + rank 48: 59.17 + rank 49: 58.79 + rank 50: 72.38 + rank 51: 58.78 + rank 52: 59.41 + rank 53: 58.80 + rank 54: 59.07 + rank 55: 59.02 + rank 56: 58.45 + rank 57: 61.16 + rank 58: 59.10 + rank 59: 58.45 + rank 60: 58.85 + rank 61: 58.47 + rank 62: 58.83 + rank 63: 58.55 + interval-time: + rank 0: 0.85 + rank 1: 0.76 + rank 2: 0.87 + rank 3: 0.78 + rank 4: 0.85 + rank 5: 0.92 + rank 6: 0.76 + rank 7: 0.76 + rank 8: 0.75 + rank 9: 0.87 + rank 10: 0.87 + rank 11: 0.83 + rank 12: 0.85 + rank 13: 0.74 + rank 14: 0.85 + rank 15: 0.84 + rank 16: 0.84 + rank 17: 0.73 + rank 18: 0.75 + rank 19: 0.85 + rank 20: 0.73 + rank 21: 0.82 + rank 22: 0.74 + rank 23: 0.81 + rank 24: 1.35 + rank 25: 1.24 + rank 26: 1.27 + rank 27: 1.44 + rank 28: 1.30 + rank 29: 1.32 + rank 30: 1.27 + rank 31: 1.40 + rank 32: 0.89 + rank 33: 0.85 + rank 34: 0.93 + rank 35: 0.97 + rank 36: 0.94 + rank 37: 0.97 + rank 38: 0.90 + rank 39: 0.91 + rank 40: 0.94 + rank 41: 0.76 + rank 42: 0.83 + rank 43: 0.76 + rank 44: 0.72 + rank 45: 0.81 + rank 46: 0.84 + rank 47: 0.81 + rank 48: 1.08 + rank 49: 1.26 + rank 50: 1.16 + rank 51: 1.28 + rank 52: 1.02 + rank 53: 1.28 + rank 54: 1.03 + rank 55: 1.07 + rank 56: 0.82 + rank 57: 0.81 + rank 58: 0.83 + rank 59: 0.83 + rank 60: 0.75 + rank 61: 0.81 + rank 62: 0.74 + rank 63: 2.84 + [2026-10-16 19:45:30] iteration 2003/ 2384 | consumed samples: 64096 | elapsed time per iteration (ms): 224538.6 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 74718.5 | valid tokens/s: 74718.5 | physical tokens/s: 75799.5 | loss tokens/s: 53656.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.497469E-01 | lm-global-supervised-tokens: 1.204804E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.454252E-02 | lm-token-buffer-mib: 2.383497E+02 | global_load_balancing_loss: 1.001234E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11152.164 | params rms: 0.060385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:45:49.209222 | finish at 2026-10-17 19:31:19 +times across ranks (ms): + forward-backward: + rank 0: 224311.70 + rank 1: 224314.39 + rank 2: 224313.58 + rank 3: 224313.55 + rank 4: 224313.28 + rank 5: 224315.69 + rank 6: 224313.08 + rank 7: 224312.27 + rank 8: 224302.06 + rank 9: 224306.06 + rank 10: 224307.05 + rank 11: 224302.30 + rank 12: 224306.72 + rank 13: 224307.67 + rank 14: 224309.16 + rank 15: 224305.56 + rank 16: 224298.48 + rank 17: 224301.00 + rank 18: 224300.52 + rank 19: 224302.44 + rank 20: 224301.75 + rank 21: 224299.89 + rank 22: 224297.08 + rank 23: 224300.59 + rank 24: 224303.44 + rank 25: 224304.94 + rank 26: 224304.11 + rank 27: 224304.17 + rank 28: 224307.55 + rank 29: 224306.33 + rank 30: 224300.86 + rank 31: 224301.81 + rank 32: 224285.91 + rank 33: 224287.61 + rank 34: 224286.31 + rank 35: 224288.20 + rank 36: 224287.66 + rank 37: 224288.27 + rank 38: 224286.80 + rank 39: 224287.64 + rank 40: 224290.98 + rank 41: 224293.22 + rank 42: 224289.55 + rank 43: 224286.33 + rank 44: 224287.41 + rank 45: 224288.52 + rank 46: 224293.69 + rank 47: 224295.69 + rank 48: 224303.44 + rank 49: 224303.62 + rank 50: 224302.31 + rank 51: 224304.12 + rank 52: 224301.28 + rank 53: 224301.47 + rank 54: 224303.39 + rank 55: 224306.73 + rank 56: 224300.62 + rank 57: 224301.06 + rank 58: 224298.28 + rank 59: 224299.41 + rank 60: 224299.70 + rank 61: 224298.88 + rank 62: 224302.06 + rank 63: 224300.53 + forward-compute: + rank 0: 117485.52 + rank 1: 117492.61 + rank 2: 117476.53 + rank 3: 117465.77 + rank 4: 117471.25 + rank 5: 117482.45 + rank 6: 117482.30 + rank 7: 117476.14 + rank 8: 117484.55 + rank 9: 117498.55 + rank 10: 117501.58 + rank 11: 117481.43 + rank 12: 117502.92 + rank 13: 117511.59 + rank 14: 117505.61 + rank 15: 117501.55 + rank 16: 117470.88 + rank 17: 117467.89 + rank 18: 117459.20 + rank 19: 117461.40 + rank 20: 117464.62 + rank 21: 117461.88 + rank 22: 117458.92 + rank 23: 117469.91 + rank 24: 117486.45 + rank 25: 117487.55 + rank 26: 117487.09 + rank 27: 117491.41 + rank 28: 117491.88 + rank 29: 117477.67 + rank 30: 117468.44 + rank 31: 117467.93 + rank 32: 62841.57 + rank 33: 62847.09 + rank 34: 62851.76 + rank 35: 62865.70 + rank 36: 62881.33 + rank 37: 62905.95 + rank 38: 62862.82 + rank 39: 62857.16 + rank 40: 62895.06 + rank 41: 62882.95 + rank 42: 62883.61 + rank 43: 62884.24 + rank 44: 62880.43 + rank 45: 62891.86 + rank 46: 62876.87 + rank 47: 62913.00 + rank 48: 62899.50 + rank 49: 62879.84 + rank 50: 62872.63 + rank 51: 62878.71 + rank 52: 62918.49 + rank 53: 62916.64 + rank 54: 62854.18 + rank 55: 62884.06 + rank 56: 62890.86 + rank 57: 62878.69 + rank 58: 62871.26 + rank 59: 62870.90 + rank 60: 62866.13 + rank 61: 62913.18 + rank 62: 62865.82 + rank 63: 62921.67 + backward-compute: + rank 0: 106780.95 + rank 1: 106770.57 + rank 2: 106785.53 + rank 3: 106795.16 + rank 4: 106789.98 + rank 5: 106782.16 + rank 6: 106779.50 + rank 7: 106788.91 + rank 8: 106764.53 + rank 9: 106751.72 + rank 10: 106748.15 + rank 11: 106767.71 + rank 12: 106747.90 + rank 13: 106740.13 + rank 14: 106743.88 + rank 15: 106751.94 + rank 16: 106782.28 + rank 17: 106783.99 + rank 18: 106792.27 + rank 19: 106790.73 + rank 20: 106788.23 + rank 21: 106789.20 + rank 22: 106792.44 + rank 23: 106785.34 + rank 24: 106761.18 + rank 25: 106759.72 + rank 26: 106754.16 + rank 27: 106756.72 + rank 28: 106749.92 + rank 29: 106757.25 + rank 30: 106783.96 + rank 31: 106771.80 + rank 32: 161408.17 + rank 33: 161400.58 + rank 34: 161397.36 + rank 35: 161381.80 + rank 36: 161367.72 + rank 37: 161343.77 + rank 38: 161386.67 + rank 39: 161394.16 + rank 40: 161360.11 + rank 41: 161371.22 + rank 42: 161371.81 + rank 43: 161368.61 + rank 44: 161371.67 + rank 45: 161362.89 + rank 46: 161378.61 + rank 47: 161347.02 + rank 48: 161368.70 + rank 49: 161389.55 + rank 50: 161393.39 + rank 51: 161390.19 + rank 52: 161349.73 + rank 53: 161346.27 + rank 54: 161413.41 + rank 55: 161387.86 + rank 56: 161377.88 + rank 57: 161386.94 + rank 58: 161391.19 + rank 59: 161393.86 + rank 60: 161400.03 + rank 61: 161351.20 + rank 62: 161401.12 + rank 63: 161344.62 + batch-generator: + rank 0: 33.24 + rank 1: 35.05 + rank 2: 35.79 + rank 3: 36.79 + rank 4: 38.38 + rank 5: 38.41 + rank 6: 36.75 + rank 7: 33.49 + rank 8: 41.57 + rank 9: 48.80 + rank 10: 44.16 + rank 11: 43.78 + rank 12: 45.57 + rank 13: 42.32 + rank 14: 49.00 + rank 15: 47.11 + rank 16: 33.29 + rank 17: 35.60 + rank 18: 34.07 + rank 19: 34.28 + rank 20: 35.28 + rank 21: 36.72 + rank 22: 33.19 + rank 23: 30.95 + rank 24: 47.78 + rank 25: 47.33 + rank 26: 51.30 + rank 27: 50.02 + rank 28: 50.07 + rank 29: 52.26 + rank 30: 44.36 + rank 31: 46.10 + rank 32: 48.30 + rank 33: 51.44 + rank 34: 48.50 + rank 35: 52.89 + rank 36: 48.63 + rank 37: 52.91 + rank 38: 41.75 + rank 39: 39.53 + rank 40: 34.81 + rank 41: 38.32 + rank 42: 37.08 + rank 43: 38.32 + rank 44: 34.83 + rank 45: 36.62 + rank 46: 34.69 + rank 47: 31.17 + rank 48: 41.38 + rank 49: 40.71 + rank 50: 43.63 + rank 51: 42.66 + rank 52: 38.94 + rank 53: 47.61 + rank 54: 35.87 + rank 55: 36.82 + rank 56: 33.75 + rank 57: 37.52 + rank 58: 36.33 + rank 59: 36.45 + rank 60: 36.87 + rank 61: 37.54 + rank 62: 36.87 + rank 63: 31.61 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.05 + rank 30: 0.06 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.03 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.89 + rank 1: 1.05 + rank 2: 0.80 + rank 3: 1.08 + rank 4: 0.93 + rank 5: 1.00 + rank 6: 1.33 + rank 7: 1.17 + rank 8: 1.11 + rank 9: 1.07 + rank 10: 0.92 + rank 11: 1.08 + rank 12: 0.92 + rank 13: 1.15 + rank 14: 1.44 + rank 15: 0.83 + rank 16: 0.90 + rank 17: 0.88 + rank 18: 0.92 + rank 19: 1.10 + rank 20: 0.83 + rank 21: 0.99 + rank 22: 0.91 + rank 23: 1.18 + rank 24: 1.70 + rank 25: 1.31 + rank 26: 1.42 + rank 27: 1.28 + rank 28: 1.25 + rank 29: 1.36 + rank 30: 1.25 + rank 31: 1.36 + rank 32: 1.41 + rank 33: 1.83 + rank 34: 1.01 + rank 35: 1.35 + rank 36: 1.22 + rank 37: 1.64 + rank 38: 1.46 + rank 39: 1.40 + rank 40: 0.83 + rank 41: 1.06 + rank 42: 1.11 + rank 43: 1.19 + rank 44: 0.84 + rank 45: 0.81 + rank 46: 1.09 + rank 47: 1.13 + rank 48: 1.31 + rank 49: 0.91 + rank 50: 0.90 + rank 51: 1.12 + rank 52: 0.83 + rank 53: 1.18 + rank 54: 1.28 + rank 55: 1.43 + rank 56: 0.88 + rank 57: 1.08 + rank 58: 0.93 + rank 59: 0.92 + rank 60: 0.81 + rank 61: 0.91 + rank 62: 1.20 + rank 63: 1.06 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.04 + rank 6: 0.03 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.25 + rank 1: 1.97 + rank 2: 1.81 + rank 3: 2.19 + rank 4: 1.72 + rank 5: 2.03 + rank 6: 1.99 + rank 7: 1.93 + rank 8: 1.79 + rank 9: 2.27 + rank 10: 1.80 + rank 11: 1.73 + rank 12: 1.95 + rank 13: 1.72 + rank 14: 2.51 + rank 15: 2.03 + rank 16: 2.37 + rank 17: 2.39 + rank 18: 2.25 + rank 19: 2.16 + rank 20: 2.25 + rank 21: 2.24 + rank 22: 2.54 + rank 23: 2.37 + rank 24: 1.86 + rank 25: 2.08 + rank 26: 2.21 + rank 27: 2.13 + rank 28: 2.36 + rank 29: 2.29 + rank 30: 2.15 + rank 31: 2.26 + rank 32: 2.00 + rank 33: 1.98 + rank 34: 1.97 + rank 35: 2.74 + rank 36: 2.11 + rank 37: 2.17 + rank 38: 2.29 + rank 39: 1.90 + rank 40: 1.72 + rank 41: 1.97 + rank 42: 1.97 + rank 43: 1.95 + rank 44: 1.87 + rank 45: 2.04 + rank 46: 1.79 + rank 47: 1.79 + rank 48: 1.96 + rank 49: 1.82 + rank 50: 2.45 + rank 51: 1.88 + rank 52: 2.10 + rank 53: 1.87 + rank 54: 1.82 + rank 55: 1.81 + rank 56: 1.75 + rank 57: 2.54 + rank 58: 1.87 + rank 59: 1.74 + rank 60: 1.81 + rank 61: 1.81 + rank 62: 1.95 + rank 63: 1.72 + optimizer-inner-step: + rank 0: 42.66 + rank 1: 42.49 + rank 2: 42.26 + rank 3: 42.47 + rank 4: 42.25 + rank 5: 42.41 + rank 6: 42.80 + rank 7: 42.74 + rank 8: 42.54 + rank 9: 42.44 + rank 10: 42.39 + rank 11: 42.47 + rank 12: 42.48 + rank 13: 42.67 + rank 14: 43.62 + rank 15: 42.32 + rank 16: 42.91 + rank 17: 42.49 + rank 18: 42.69 + rank 19: 42.49 + rank 20: 42.41 + rank 21: 42.39 + rank 22: 43.04 + rank 23: 42.64 + rank 24: 42.18 + rank 25: 42.44 + rank 26: 42.24 + rank 27: 42.52 + rank 28: 46.44 + rank 29: 42.17 + rank 30: 42.37 + rank 31: 42.43 + rank 32: 42.53 + rank 33: 42.43 + rank 34: 42.54 + rank 35: 51.66 + rank 36: 42.95 + rank 37: 42.55 + rank 38: 42.32 + rank 39: 42.31 + rank 40: 42.48 + rank 41: 42.55 + rank 42: 42.59 + rank 43: 42.69 + rank 44: 42.24 + rank 45: 42.83 + rank 46: 42.48 + rank 47: 42.18 + rank 48: 42.44 + rank 49: 42.44 + rank 50: 54.32 + rank 51: 42.22 + rank 52: 42.53 + rank 53: 42.39 + rank 54: 42.56 + rank 55: 42.49 + rank 56: 42.16 + rank 57: 47.65 + rank 58: 42.44 + rank 59: 42.26 + rank 60: 42.62 + rank 61: 42.27 + rank 62: 42.30 + rank 63: 42.21 + optimizer-copy-main-to-model-params: + rank 0: 4.41 + rank 1: 4.18 + rank 2: 3.76 + rank 3: 4.36 + rank 4: 3.80 + rank 5: 4.42 + rank 6: 4.15 + rank 7: 4.12 + rank 8: 3.85 + rank 9: 4.59 + rank 10: 3.90 + rank 11: 3.78 + rank 12: 3.98 + rank 13: 3.80 + rank 14: 4.96 + rank 15: 4.21 + rank 16: 4.46 + rank 17: 3.98 + rank 18: 4.02 + rank 19: 4.02 + rank 20: 4.08 + rank 21: 4.13 + rank 22: 4.58 + rank 23: 4.19 + rank 24: 3.85 + rank 25: 4.20 + rank 26: 4.01 + rank 27: 4.05 + rank 28: 4.51 + rank 29: 4.36 + rank 30: 4.32 + rank 31: 4.04 + rank 32: 4.27 + rank 33: 4.10 + rank 34: 4.01 + rank 35: 5.27 + rank 36: 4.36 + rank 37: 4.54 + rank 38: 4.59 + rank 39: 3.97 + rank 40: 3.81 + rank 41: 4.18 + rank 42: 3.99 + rank 43: 4.00 + rank 44: 4.02 + rank 45: 4.30 + rank 46: 3.77 + rank 47: 3.76 + rank 48: 4.05 + rank 49: 3.95 + rank 50: 5.03 + rank 51: 3.97 + rank 52: 4.21 + rank 53: 3.95 + rank 54: 3.92 + rank 55: 3.92 + rank 56: 3.78 + rank 57: 4.71 + rank 58: 4.02 + rank 59: 3.79 + rank 60: 3.80 + rank 61: 3.77 + rank 62: 3.95 + rank 63: 3.65 + optimizer: + rank 0: 54.44 + rank 1: 53.87 + rank 2: 53.09 + rank 3: 54.10 + rank 4: 53.11 + rank 5: 54.10 + rank 6: 54.18 + rank 7: 54.13 + rank 8: 53.51 + rank 9: 54.42 + rank 10: 53.42 + rank 11: 53.34 + rank 12: 53.63 + rank 13: 53.57 + rank 14: 56.19 + rank 15: 53.76 + rank 16: 54.89 + rank 17: 53.89 + rank 18: 54.07 + rank 19: 53.90 + rank 20: 53.95 + rank 21: 54.00 + rank 22: 55.20 + rank 23: 54.35 + rank 24: 53.17 + rank 25: 53.97 + rank 26: 53.52 + rank 27: 53.81 + rank 28: 58.27 + rank 29: 53.97 + rank 30: 53.98 + rank 31: 53.82 + rank 32: 54.05 + rank 33: 53.65 + rank 34: 53.67 + rank 35: 64.51 + rank 36: 54.64 + rank 37: 54.37 + rank 38: 54.25 + rank 39: 53.46 + rank 40: 53.38 + rank 41: 53.93 + rank 42: 53.76 + rank 43: 53.84 + rank 44: 53.43 + rank 45: 54.44 + rank 46: 53.32 + rank 47: 53.04 + rank 48: 53.68 + rank 49: 53.46 + rank 50: 66.83 + rank 51: 53.31 + rank 52: 53.95 + rank 53: 53.45 + rank 54: 53.57 + rank 55: 53.48 + rank 56: 53.02 + rank 57: 59.95 + rank 58: 53.61 + rank 59: 53.14 + rank 60: 53.48 + rank 61: 53.09 + rank 62: 53.41 + rank 63: 52.93 + interval-time: + rank 0: 0.83 + rank 1: 0.84 + rank 2: 0.85 + rank 3: 0.75 + rank 4: 0.86 + rank 5: 0.93 + rank 6: 0.78 + rank 7: 0.78 + rank 8: 7.34 + rank 9: 7.37 + rank 10: 7.37 + rank 11: 7.36 + rank 12: 7.35 + rank 13: 7.36 + rank 14: 7.34 + rank 15: 7.33 + rank 16: 8.23 + rank 17: 8.22 + rank 18: 8.22 + rank 19: 8.26 + rank 20: 8.22 + rank 21: 8.22 + rank 22: 8.21 + rank 23: 8.22 + rank 24: 0.95 + rank 25: 0.96 + rank 26: 0.91 + rank 27: 0.96 + rank 28: 0.93 + rank 29: 0.95 + rank 30: 1.01 + rank 31: 0.95 + rank 32: 0.84 + rank 33: 0.87 + rank 34: 0.85 + rank 35: 1.07 + rank 36: 0.92 + rank 37: 0.83 + rank 38: 0.91 + rank 39: 0.79 + rank 40: 0.85 + rank 41: 0.85 + rank 42: 0.84 + rank 43: 0.76 + rank 44: 0.72 + rank 45: 0.83 + rank 46: 0.77 + rank 47: 0.82 + rank 48: 0.93 + rank 49: 0.99 + rank 50: 0.85 + rank 51: 0.95 + rank 52: 0.82 + rank 53: 0.98 + rank 54: 0.91 + rank 55: 0.91 + rank 56: 0.83 + rank 57: 0.83 + rank 58: 0.84 + rank 59: 0.73 + rank 60: 0.73 + rank 61: 0.82 + rank 62: 0.72 + rank 63: 2.88 + [2026-10-16 19:49:12] iteration 2004/ 2384 | consumed samples: 64128 | elapsed time per iteration (ms): 222299.8 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 75471.1 | valid tokens/s: 75471.1 | physical tokens/s: 75695.1 | loss tokens/s: 53406.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.764616E-01 | lm-global-supervised-tokens: 1.187218E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.422352E-02 | lm-token-buffer-mib: 2.328670E+02 | global_load_balancing_loss: 1.000747E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11152.136 | params rms: 0.060385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:27:53.922791 | finish at 2026-10-17 19:17:06 +times across ranks (ms): + forward-backward: + rank 0: 222070.88 + rank 1: 222066.81 + rank 2: 222071.23 + rank 3: 222067.58 + rank 4: 222067.92 + rank 5: 222071.08 + rank 6: 222068.14 + rank 7: 222072.19 + rank 8: 222059.19 + rank 9: 222055.22 + rank 10: 222059.33 + rank 11: 222059.64 + rank 12: 222058.39 + rank 13: 222061.08 + rank 14: 222058.81 + rank 15: 222058.19 + rank 16: 222050.34 + rank 17: 222050.08 + rank 18: 222051.59 + rank 19: 222053.11 + rank 20: 222053.56 + rank 21: 222053.33 + rank 22: 222051.39 + rank 23: 222051.19 + rank 24: 222057.84 + rank 25: 222062.25 + rank 26: 222058.66 + rank 27: 222062.84 + rank 28: 222061.27 + rank 29: 222065.38 + rank 30: 222056.73 + rank 31: 222057.33 + rank 32: 222054.72 + rank 33: 222050.64 + rank 34: 222054.84 + rank 35: 222051.02 + rank 36: 222051.30 + rank 37: 222054.44 + rank 38: 222051.44 + rank 39: 222055.72 + rank 40: 222048.92 + rank 41: 222044.50 + rank 42: 222046.95 + rank 43: 222046.92 + rank 44: 222044.11 + rank 45: 222048.16 + rank 46: 222050.67 + rank 47: 222050.09 + rank 48: 222056.94 + rank 49: 222055.70 + rank 50: 222056.70 + rank 51: 222058.39 + rank 52: 222055.55 + rank 53: 222056.64 + rank 54: 222058.78 + rank 55: 222059.91 + rank 56: 222058.70 + rank 57: 222059.69 + rank 58: 222057.73 + rank 59: 222058.53 + rank 60: 222056.67 + rank 61: 222056.94 + rank 62: 222061.27 + rank 63: 222060.77 + forward-compute: + rank 0: 121378.35 + rank 1: 121370.63 + rank 2: 121368.64 + rank 3: 121367.38 + rank 4: 121370.44 + rank 5: 121370.80 + rank 6: 121362.93 + rank 7: 121366.48 + rank 8: 121412.57 + rank 9: 121408.31 + rank 10: 121413.87 + rank 11: 121408.91 + rank 12: 121409.49 + rank 13: 121416.54 + rank 14: 121414.30 + rank 15: 121409.64 + rank 16: 121364.23 + rank 17: 121358.64 + rank 18: 121362.34 + rank 19: 121362.73 + rank 20: 121366.90 + rank 21: 121368.16 + rank 22: 121356.02 + rank 23: 121357.38 + rank 24: 121366.16 + rank 25: 121366.70 + rank 26: 121365.13 + rank 27: 121368.92 + rank 28: 121367.68 + rank 29: 121363.58 + rank 30: 121359.02 + rank 31: 121357.34 + rank 32: 61165.70 + rank 33: 61167.77 + rank 34: 61173.12 + rank 35: 61183.14 + rank 36: 61201.87 + rank 37: 61214.48 + rank 38: 61187.21 + rank 39: 61190.99 + rank 40: 61212.42 + rank 41: 61169.65 + rank 42: 61187.74 + rank 43: 61190.68 + rank 44: 61181.39 + rank 45: 61210.62 + rank 46: 61154.30 + rank 47: 61194.95 + rank 48: 61167.38 + rank 49: 61157.47 + rank 50: 61160.91 + rank 51: 61148.94 + rank 52: 61198.15 + rank 53: 61185.60 + rank 54: 61141.40 + rank 55: 61169.89 + rank 56: 61184.43 + rank 57: 61172.94 + rank 58: 61156.80 + rank 59: 61153.41 + rank 60: 61188.04 + rank 61: 61182.12 + rank 62: 61171.75 + rank 63: 61214.13 + backward-compute: + rank 0: 100636.80 + rank 1: 100637.49 + rank 2: 100644.84 + rank 3: 100642.50 + rank 4: 100642.93 + rank 5: 100643.03 + rank 6: 100644.95 + rank 7: 100648.83 + rank 8: 100598.29 + rank 9: 100590.76 + rank 10: 100589.19 + rank 11: 100599.65 + rank 12: 100595.65 + rank 13: 100592.33 + rank 14: 100592.34 + rank 15: 100601.22 + rank 16: 100641.67 + rank 17: 100640.30 + rank 18: 100641.85 + rank 19: 100637.81 + rank 20: 100636.99 + rank 21: 100634.25 + rank 22: 100645.39 + rank 23: 100647.01 + rank 24: 100637.15 + rank 25: 100633.84 + rank 26: 100631.34 + rank 27: 100630.30 + rank 28: 100626.75 + rank 29: 100631.34 + rank 30: 100652.34 + rank 31: 100643.91 + rank 32: 160842.56 + rank 33: 160834.66 + rank 34: 160835.25 + rank 35: 160818.58 + rank 36: 160804.14 + rank 37: 160792.59 + rank 38: 160817.28 + rank 39: 160819.39 + rank 40: 160799.14 + rank 41: 160838.97 + rank 42: 160825.12 + rank 43: 160821.89 + rank 44: 160829.55 + rank 45: 160802.27 + rank 46: 160859.06 + rank 47: 160820.31 + rank 48: 160854.33 + rank 49: 160862.95 + rank 50: 160857.53 + rank 51: 160872.44 + rank 52: 160825.11 + rank 53: 160833.08 + rank 54: 160878.30 + rank 55: 160854.89 + rank 56: 160842.78 + rank 57: 160851.08 + rank 58: 160864.94 + rank 59: 160868.05 + rank 60: 160833.12 + rank 61: 160840.16 + rank 62: 160854.33 + rank 63: 160810.48 + batch-generator: + rank 0: 33.90 + rank 1: 37.29 + rank 2: 38.25 + rank 3: 38.32 + rank 4: 37.74 + rank 5: 39.10 + rank 6: 38.10 + rank 7: 35.09 + rank 8: 51.23 + rank 9: 52.52 + rank 10: 49.25 + rank 11: 51.09 + rank 12: 51.30 + rank 13: 49.58 + rank 14: 52.74 + rank 15: 47.13 + rank 16: 35.17 + rank 17: 35.99 + rank 18: 35.62 + rank 19: 35.70 + rank 20: 34.82 + rank 21: 36.04 + rank 22: 35.08 + rank 23: 33.01 + rank 24: 46.69 + rank 25: 47.67 + rank 26: 48.33 + rank 27: 49.36 + rank 28: 47.31 + rank 29: 49.19 + rank 30: 44.03 + rank 31: 43.44 + rank 32: 50.85 + rank 33: 51.37 + rank 34: 49.27 + rank 35: 53.89 + rank 36: 50.25 + rank 37: 50.33 + rank 38: 42.66 + rank 39: 43.47 + rank 40: 32.89 + rank 41: 39.92 + rank 42: 40.09 + rank 43: 36.31 + rank 44: 34.98 + rank 45: 34.93 + rank 46: 39.14 + rank 47: 31.99 + rank 48: 43.47 + rank 49: 42.01 + rank 50: 46.39 + rank 51: 43.45 + rank 52: 34.30 + rank 53: 44.10 + rank 54: 38.68 + rank 55: 37.97 + rank 56: 30.80 + rank 57: 37.68 + rank 58: 41.21 + rank 59: 39.61 + rank 60: 37.55 + rank 61: 35.91 + rank 62: 35.95 + rank 63: 30.30 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.04 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.03 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.04 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.04 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.04 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.13 + rank 1: 0.91 + rank 2: 1.16 + rank 3: 1.30 + rank 4: 1.08 + rank 5: 1.25 + rank 6: 1.08 + rank 7: 1.21 + rank 8: 1.23 + rank 9: 0.90 + rank 10: 0.82 + rank 11: 1.37 + rank 12: 0.85 + rank 13: 0.80 + rank 14: 1.30 + rank 15: 0.90 + rank 16: 1.20 + rank 17: 0.82 + rank 18: 0.92 + rank 19: 1.00 + rank 20: 0.93 + rank 21: 0.90 + rank 22: 0.88 + rank 23: 0.90 + rank 24: 1.24 + rank 25: 1.21 + rank 26: 1.25 + rank 27: 2.05 + rank 28: 1.90 + rank 29: 1.70 + rank 30: 1.12 + rank 31: 1.33 + rank 32: 1.72 + rank 33: 1.25 + rank 34: 1.67 + rank 35: 1.22 + rank 36: 1.29 + rank 37: 1.26 + rank 38: 1.36 + rank 39: 1.19 + rank 40: 0.91 + rank 41: 1.20 + rank 42: 1.22 + rank 43: 0.79 + rank 44: 1.09 + rank 45: 1.34 + rank 46: 1.31 + rank 47: 1.35 + rank 48: 1.26 + rank 49: 1.11 + rank 50: 1.04 + rank 51: 1.31 + rank 52: 0.83 + rank 53: 1.25 + rank 54: 0.93 + rank 55: 1.37 + rank 56: 0.82 + rank 57: 1.13 + rank 58: 0.90 + rank 59: 0.93 + rank 60: 0.93 + rank 61: 0.86 + rank 62: 0.95 + rank 63: 1.28 + params-all-gather: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.02 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.03 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.03 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.24 + rank 1: 1.88 + rank 2: 1.79 + rank 3: 2.23 + rank 4: 1.74 + rank 5: 1.95 + rank 6: 1.93 + rank 7: 1.97 + rank 8: 1.88 + rank 9: 2.24 + rank 10: 1.94 + rank 11: 1.85 + rank 12: 1.92 + rank 13: 1.47 + rank 14: 1.99 + rank 15: 2.15 + rank 16: 2.51 + rank 17: 2.25 + rank 18: 2.29 + rank 19: 2.23 + rank 20: 2.29 + rank 21: 2.35 + rank 22: 2.73 + rank 23: 2.46 + rank 24: 2.14 + rank 25: 2.14 + rank 26: 2.13 + rank 27: 2.28 + rank 28: 2.55 + rank 29: 2.54 + rank 30: 2.26 + rank 31: 2.24 + rank 32: 2.04 + rank 33: 1.90 + rank 34: 2.03 + rank 35: 2.22 + rank 36: 2.08 + rank 37: 1.99 + rank 38: 2.32 + rank 39: 1.99 + rank 40: 1.83 + rank 41: 1.92 + rank 42: 1.95 + rank 43: 1.95 + rank 44: 1.91 + rank 45: 2.12 + rank 46: 1.74 + rank 47: 1.75 + rank 48: 1.93 + rank 49: 1.82 + rank 50: 2.50 + rank 51: 1.90 + rank 52: 1.98 + rank 53: 1.87 + rank 54: 1.83 + rank 55: 1.86 + rank 56: 1.86 + rank 57: 2.46 + rank 58: 1.91 + rank 59: 1.79 + rank 60: 1.82 + rank 61: 1.75 + rank 62: 1.96 + rank 63: 1.80 + optimizer-inner-step: + rank 0: 48.69 + rank 1: 48.67 + rank 2: 48.40 + rank 3: 48.58 + rank 4: 48.40 + rank 5: 48.60 + rank 6: 48.92 + rank 7: 48.84 + rank 8: 48.67 + rank 9: 48.59 + rank 10: 48.52 + rank 11: 48.63 + rank 12: 48.65 + rank 13: 48.86 + rank 14: 48.46 + rank 15: 48.45 + rank 16: 48.98 + rank 17: 48.65 + rank 18: 48.77 + rank 19: 48.65 + rank 20: 48.60 + rank 21: 48.49 + rank 22: 49.15 + rank 23: 48.71 + rank 24: 48.33 + rank 25: 48.66 + rank 26: 48.55 + rank 27: 48.70 + rank 28: 48.99 + rank 29: 48.51 + rank 30: 48.57 + rank 31: 48.71 + rank 32: 48.20 + rank 33: 48.16 + rank 34: 48.58 + rank 35: 50.00 + rank 36: 48.73 + rank 37: 48.25 + rank 38: 48.51 + rank 39: 48.42 + rank 40: 48.57 + rank 41: 48.66 + rank 42: 48.64 + rank 43: 48.76 + rank 44: 48.33 + rank 45: 49.01 + rank 46: 48.66 + rank 47: 48.35 + rank 48: 48.61 + rank 49: 48.54 + rank 50: 54.69 + rank 51: 48.34 + rank 52: 48.65 + rank 53: 48.48 + rank 54: 48.73 + rank 55: 55.75 + rank 56: 48.23 + rank 57: 49.45 + rank 58: 48.51 + rank 59: 48.39 + rank 60: 48.73 + rank 61: 48.42 + rank 62: 48.48 + rank 63: 48.33 + optimizer-copy-main-to-model-params: + rank 0: 4.36 + rank 1: 3.91 + rank 2: 3.74 + rank 3: 4.67 + rank 4: 3.74 + rank 5: 4.06 + rank 6: 4.10 + rank 7: 4.07 + rank 8: 3.87 + rank 9: 4.50 + rank 10: 3.92 + rank 11: 3.83 + rank 12: 3.95 + rank 13: 3.78 + rank 14: 4.01 + rank 15: 4.22 + rank 16: 4.42 + rank 17: 3.98 + rank 18: 4.05 + rank 19: 4.01 + rank 20: 4.02 + rank 21: 4.29 + rank 22: 4.70 + rank 23: 4.30 + rank 24: 3.90 + rank 25: 4.25 + rank 26: 3.95 + rank 27: 4.10 + rank 28: 4.38 + rank 29: 4.43 + rank 30: 4.39 + rank 31: 4.03 + rank 32: 4.22 + rank 33: 4.00 + rank 34: 4.02 + rank 35: 4.59 + rank 36: 4.22 + rank 37: 4.15 + rank 38: 4.59 + rank 39: 4.00 + rank 40: 3.79 + rank 41: 4.06 + rank 42: 3.97 + rank 43: 4.03 + rank 44: 3.93 + rank 45: 4.26 + rank 46: 3.77 + rank 47: 3.72 + rank 48: 4.08 + rank 49: 3.88 + rank 50: 5.01 + rank 51: 3.98 + rank 52: 4.20 + rank 53: 3.90 + rank 54: 3.87 + rank 55: 3.95 + rank 56: 3.97 + rank 57: 4.78 + rank 58: 3.94 + rank 59: 3.75 + rank 60: 3.75 + rank 61: 3.76 + rank 62: 3.98 + rank 63: 3.77 + optimizer: + rank 0: 60.73 + rank 1: 59.98 + rank 2: 59.44 + rank 3: 60.87 + rank 4: 59.44 + rank 5: 60.04 + rank 6: 60.43 + rank 7: 60.35 + rank 8: 59.91 + rank 9: 60.69 + rank 10: 59.79 + rank 11: 59.80 + rank 12: 59.98 + rank 13: 59.95 + rank 14: 59.99 + rank 15: 60.16 + rank 16: 61.23 + rank 17: 60.26 + rank 18: 60.42 + rank 19: 60.30 + rank 20: 60.27 + rank 21: 60.52 + rank 22: 61.71 + rank 23: 60.79 + rank 24: 59.61 + rank 25: 60.36 + rank 26: 59.87 + rank 27: 60.19 + rank 28: 60.92 + rank 29: 60.48 + rank 30: 60.44 + rank 31: 60.15 + rank 32: 60.20 + rank 33: 59.78 + rank 34: 59.96 + rank 35: 62.43 + rank 36: 60.37 + rank 37: 60.13 + rank 38: 60.64 + rank 39: 59.77 + rank 40: 59.75 + rank 41: 60.14 + rank 42: 60.03 + rank 43: 60.20 + rank 44: 59.68 + rank 45: 60.76 + rank 46: 59.72 + rank 47: 59.37 + rank 48: 60.08 + rank 49: 59.73 + rank 50: 67.43 + rank 51: 59.66 + rank 52: 60.29 + rank 53: 59.70 + rank 54: 59.91 + rank 55: 67.02 + rank 56: 59.55 + rank 57: 62.00 + rank 58: 59.85 + rank 59: 59.44 + rank 60: 59.78 + rank 61: 59.50 + rank 62: 59.87 + rank 63: 59.44 + interval-time: + rank 0: 0.82 + rank 1: 0.83 + rank 2: 0.84 + rank 3: 0.76 + rank 4: 0.85 + rank 5: 0.87 + rank 6: 0.77 + rank 7: 0.72 + rank 8: 0.74 + rank 9: 0.81 + rank 10: 0.84 + rank 11: 0.83 + rank 12: 0.84 + rank 13: 0.70 + rank 14: 0.84 + rank 15: 0.80 + rank 16: 0.83 + rank 17: 0.82 + rank 18: 0.78 + rank 19: 0.84 + rank 20: 0.73 + rank 21: 0.83 + rank 22: 0.76 + rank 23: 0.83 + rank 24: 1.36 + rank 25: 1.28 + rank 26: 1.35 + rank 27: 1.20 + rank 28: 1.27 + rank 29: 1.38 + rank 30: 1.38 + rank 31: 1.30 + rank 32: 0.86 + rank 33: 0.84 + rank 34: 0.84 + rank 35: 0.96 + rank 36: 0.92 + rank 37: 0.84 + rank 38: 0.90 + rank 39: 0.80 + rank 40: 0.84 + rank 41: 0.85 + rank 42: 0.84 + rank 43: 0.74 + rank 44: 0.73 + rank 45: 0.83 + rank 46: 0.74 + rank 47: 0.79 + rank 48: 0.92 + rank 49: 0.98 + rank 50: 0.90 + rank 51: 0.94 + rank 52: 0.81 + rank 53: 0.97 + rank 54: 0.90 + rank 55: 0.91 + rank 56: 0.81 + rank 57: 0.82 + rank 58: 0.83 + rank 59: 0.73 + rank 60: 0.75 + rank 61: 0.83 + rank 62: 0.74 + rank 63: 2.35 + [2026-10-16 19:52:52] iteration 2005/ 2384 | consumed samples: 64160 | elapsed time per iteration (ms): 220002.9 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 76259.0 | valid tokens/s: 76259.0 | physical tokens/s: 76784.7 | loss tokens/s: 57952.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.056471E-01 | lm-global-supervised-tokens: 1.274966E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.235823E-02 | lm-token-buffer-mib: 2.327853E+02 | global_load_balancing_loss: 9.996746E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11152.110 | params rms: 0.060385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:09:41.092822 | finish at 2026-10-17 19:02:33 +times across ranks (ms): + forward-backward: + rank 0: 219771.39 + rank 1: 219771.30 + rank 2: 219768.09 + rank 3: 219771.75 + rank 4: 219776.28 + rank 5: 219773.19 + rank 6: 219770.66 + rank 7: 219772.83 + rank 8: 219769.53 + rank 9: 219770.25 + rank 10: 219771.28 + rank 11: 219768.50 + rank 12: 219768.23 + rank 13: 219773.75 + rank 14: 219765.81 + rank 15: 219770.59 + rank 16: 219763.27 + rank 17: 219761.91 + rank 18: 219763.44 + rank 19: 219762.69 + rank 20: 219765.61 + rank 21: 219766.22 + rank 22: 219765.88 + rank 23: 219766.69 + rank 24: 219766.45 + rank 25: 219765.86 + rank 26: 219763.42 + rank 27: 219764.06 + rank 28: 219765.25 + rank 29: 219765.00 + rank 30: 219766.33 + rank 31: 219766.80 + rank 32: 219754.06 + rank 33: 219754.22 + rank 34: 219749.17 + rank 35: 219753.08 + rank 36: 219757.17 + rank 37: 219753.22 + rank 38: 219752.97 + rank 39: 219755.78 + rank 40: 219759.70 + rank 41: 219757.91 + rank 42: 219757.36 + rank 43: 219754.47 + rank 44: 219752.73 + rank 45: 219756.17 + rank 46: 219759.06 + rank 47: 219761.16 + rank 48: 219766.45 + rank 49: 219764.44 + rank 50: 219765.17 + rank 51: 219765.66 + rank 52: 219765.44 + rank 53: 219768.72 + rank 54: 219769.55 + rank 55: 219770.94 + rank 56: 219769.19 + rank 57: 219768.20 + rank 58: 219765.73 + rank 59: 219765.23 + rank 60: 219765.84 + rank 61: 219765.70 + rank 62: 219771.52 + rank 63: 219770.03 + forward-compute: + rank 0: 117276.18 + rank 1: 117275.50 + rank 2: 117241.73 + rank 3: 117234.30 + rank 4: 117235.01 + rank 5: 117243.16 + rank 6: 117250.16 + rank 7: 117258.91 + rank 8: 117193.77 + rank 9: 117203.16 + rank 10: 117194.29 + rank 11: 117194.47 + rank 12: 117205.10 + rank 13: 117211.28 + rank 14: 117203.00 + rank 15: 117211.26 + rank 16: 117229.16 + rank 17: 117223.30 + rank 18: 117222.56 + rank 19: 117220.72 + rank 20: 117217.12 + rank 21: 117219.88 + rank 22: 117223.10 + rank 23: 117222.06 + rank 24: 117142.95 + rank 25: 117136.58 + rank 26: 117133.34 + rank 27: 117131.52 + rank 28: 117154.42 + rank 29: 117129.33 + rank 30: 117129.62 + rank 31: 117125.47 + rank 32: 61003.92 + rank 33: 61006.30 + rank 34: 61015.40 + rank 35: 61031.47 + rank 36: 61062.47 + rank 37: 61065.69 + rank 38: 61041.61 + rank 39: 61038.70 + rank 40: 61070.79 + rank 41: 61035.90 + rank 42: 61044.31 + rank 43: 61039.79 + rank 44: 61030.44 + rank 45: 61057.34 + rank 46: 61024.75 + rank 47: 61040.95 + rank 48: 61004.63 + rank 49: 61016.01 + rank 50: 61011.80 + rank 51: 60999.89 + rank 52: 61042.17 + rank 53: 61022.55 + rank 54: 60986.11 + rank 55: 61013.09 + rank 56: 61026.43 + rank 57: 61020.43 + rank 58: 61017.10 + rank 59: 60991.08 + rank 60: 61010.00 + rank 61: 61024.74 + rank 62: 61002.68 + rank 63: 61048.03 + backward-compute: + rank 0: 102448.45 + rank 1: 102442.23 + rank 2: 102475.30 + rank 3: 102484.66 + rank 4: 102488.46 + rank 5: 102479.96 + rank 6: 102471.77 + rank 7: 102466.25 + rank 8: 102527.79 + rank 9: 102514.18 + rank 10: 102524.03 + rank 11: 102525.38 + rank 12: 102510.88 + rank 13: 102512.29 + rank 14: 102508.96 + rank 15: 102513.02 + rank 16: 102461.02 + rank 17: 102460.68 + rank 18: 102465.49 + rank 19: 102462.39 + rank 20: 102471.63 + rank 21: 102467.53 + rank 22: 102466.09 + rank 23: 102467.38 + rank 24: 102527.06 + rank 25: 102530.44 + rank 26: 102530.83 + rank 27: 102528.87 + rank 28: 102502.23 + rank 29: 102525.94 + rank 30: 102547.23 + rank 31: 102541.89 + rank 32: 158713.67 + rank 33: 158708.42 + rank 34: 158695.88 + rank 35: 158681.69 + rank 36: 158655.88 + rank 37: 158649.42 + rank 38: 158672.80 + rank 39: 158680.25 + rank 40: 158655.80 + rank 41: 158686.70 + rank 42: 158679.53 + rank 43: 158681.11 + rank 44: 158689.09 + rank 45: 158664.72 + rank 46: 158698.59 + rank 47: 158686.25 + rank 48: 158727.39 + rank 49: 158713.39 + rank 50: 158717.67 + rank 51: 158729.42 + rank 52: 158691.17 + rank 53: 158707.78 + rank 54: 158746.98 + rank 55: 158721.69 + rank 56: 158710.75 + rank 57: 158712.53 + rank 58: 158712.97 + rank 59: 158738.48 + rank 60: 158720.73 + rank 61: 158707.02 + rank 62: 158733.67 + rank 63: 158688.39 + batch-generator: + rank 0: 34.56 + rank 1: 37.80 + rank 2: 38.03 + rank 3: 38.30 + rank 4: 39.60 + rank 5: 37.57 + rank 6: 39.18 + rank 7: 36.82 + rank 8: 62.53 + rank 9: 66.39 + rank 10: 61.71 + rank 11: 62.88 + rank 12: 64.22 + rank 13: 66.42 + rank 14: 68.30 + rank 15: 66.48 + rank 16: 44.82 + rank 17: 46.62 + rank 18: 46.25 + rank 19: 47.40 + rank 20: 46.51 + rank 21: 48.65 + rank 22: 47.08 + rank 23: 43.82 + rank 24: 45.61 + rank 25: 45.77 + rank 26: 51.98 + rank 27: 46.96 + rank 28: 49.04 + rank 29: 51.99 + rank 30: 44.11 + rank 31: 46.23 + rank 32: 44.52 + rank 33: 47.87 + rank 34: 47.93 + rank 35: 49.11 + rank 36: 46.57 + rank 37: 45.75 + rank 38: 42.00 + rank 39: 39.28 + rank 40: 32.89 + rank 41: 37.10 + rank 42: 35.24 + rank 43: 36.44 + rank 44: 36.22 + rank 45: 35.65 + rank 46: 37.28 + rank 47: 29.25 + rank 48: 41.93 + rank 49: 42.73 + rank 50: 44.55 + rank 51: 45.14 + rank 52: 38.61 + rank 53: 42.02 + rank 54: 35.30 + rank 55: 38.00 + rank 56: 31.42 + rank 57: 37.91 + rank 58: 39.07 + rank 59: 38.27 + rank 60: 38.36 + rank 61: 36.85 + rank 62: 35.12 + rank 63: 31.41 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.04 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.86 + rank 1: 0.87 + rank 2: 1.03 + rank 3: 1.66 + rank 4: 0.86 + rank 5: 0.87 + rank 6: 0.82 + rank 7: 0.88 + rank 8: 0.86 + rank 9: 1.09 + rank 10: 1.55 + rank 11: 1.01 + rank 12: 0.83 + rank 13: 0.86 + rank 14: 1.03 + rank 15: 0.79 + rank 16: 1.15 + rank 17: 1.26 + rank 18: 0.97 + rank 19: 0.86 + rank 20: 1.14 + rank 21: 0.84 + rank 22: 0.88 + rank 23: 0.81 + rank 24: 1.30 + rank 25: 1.42 + rank 26: 1.46 + rank 27: 1.22 + rank 28: 1.30 + rank 29: 1.43 + rank 30: 1.17 + rank 31: 1.27 + rank 32: 1.47 + rank 33: 1.44 + rank 34: 1.52 + rank 35: 1.78 + rank 36: 1.55 + rank 37: 1.47 + rank 38: 1.28 + rank 39: 1.19 + rank 40: 0.85 + rank 41: 0.84 + rank 42: 0.85 + rank 43: 0.87 + rank 44: 0.83 + rank 45: 1.11 + rank 46: 1.02 + rank 47: 0.86 + rank 48: 1.15 + rank 49: 1.09 + rank 50: 0.90 + rank 51: 1.45 + rank 52: 0.87 + rank 53: 1.11 + rank 54: 1.14 + rank 55: 1.06 + rank 56: 0.95 + rank 57: 0.94 + rank 58: 0.85 + rank 59: 0.86 + rank 60: 0.80 + rank 61: 1.16 + rank 62: 0.99 + rank 63: 0.97 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.03 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.22 + rank 1: 1.81 + rank 2: 1.75 + rank 3: 2.16 + rank 4: 1.71 + rank 5: 1.92 + rank 6: 1.90 + rank 7: 1.91 + rank 8: 1.89 + rank 9: 2.14 + rank 10: 1.79 + rank 11: 1.81 + rank 12: 1.88 + rank 13: 1.83 + rank 14: 1.91 + rank 15: 2.12 + rank 16: 2.47 + rank 17: 2.20 + rank 18: 2.27 + rank 19: 2.18 + rank 20: 2.21 + rank 21: 2.35 + rank 22: 2.61 + rank 23: 2.33 + rank 24: 1.98 + rank 25: 2.28 + rank 26: 2.03 + rank 27: 2.31 + rank 28: 2.51 + rank 29: 2.50 + rank 30: 2.26 + rank 31: 2.31 + rank 32: 2.08 + rank 33: 1.96 + rank 34: 2.00 + rank 35: 2.25 + rank 36: 2.09 + rank 37: 2.03 + rank 38: 2.31 + rank 39: 2.00 + rank 40: 1.77 + rank 41: 1.90 + rank 42: 1.90 + rank 43: 1.93 + rank 44: 1.86 + rank 45: 2.07 + rank 46: 1.75 + rank 47: 1.67 + rank 48: 1.95 + rank 49: 1.82 + rank 50: 2.48 + rank 51: 1.82 + rank 52: 1.95 + rank 53: 1.89 + rank 54: 1.86 + rank 55: 1.86 + rank 56: 1.81 + rank 57: 2.39 + rank 58: 1.82 + rank 59: 1.81 + rank 60: 1.78 + rank 61: 1.73 + rank 62: 1.87 + rank 63: 1.77 + optimizer-inner-step: + rank 0: 43.96 + rank 1: 43.85 + rank 2: 43.59 + rank 3: 43.71 + rank 4: 43.58 + rank 5: 43.75 + rank 6: 44.09 + rank 7: 43.95 + rank 8: 43.85 + rank 9: 43.86 + rank 10: 43.60 + rank 11: 43.79 + rank 12: 43.84 + rank 13: 44.01 + rank 14: 43.82 + rank 15: 43.63 + rank 16: 44.18 + rank 17: 43.89 + rank 18: 44.00 + rank 19: 43.87 + rank 20: 43.84 + rank 21: 43.74 + rank 22: 44.32 + rank 23: 44.01 + rank 24: 39.82 + rank 25: 44.24 + rank 26: 41.62 + rank 27: 45.42 + rank 28: 49.36 + rank 29: 44.39 + rank 30: 43.73 + rank 31: 45.42 + rank 32: 43.90 + rank 33: 43.74 + rank 34: 43.88 + rank 35: 48.74 + rank 36: 43.89 + rank 37: 43.83 + rank 38: 43.79 + rank 39: 43.74 + rank 40: 43.75 + rank 41: 43.93 + rank 42: 43.91 + rank 43: 44.02 + rank 44: 43.60 + rank 45: 44.23 + rank 46: 43.85 + rank 47: 43.53 + rank 48: 43.76 + rank 49: 43.59 + rank 50: 54.64 + rank 51: 43.47 + rank 52: 43.85 + rank 53: 43.65 + rank 54: 43.84 + rank 55: 43.75 + rank 56: 43.49 + rank 57: 46.59 + rank 58: 43.72 + rank 59: 43.61 + rank 60: 43.92 + rank 61: 43.61 + rank 62: 43.73 + rank 63: 43.50 + optimizer-copy-main-to-model-params: + rank 0: 4.46 + rank 1: 3.85 + rank 2: 3.77 + rank 3: 4.38 + rank 4: 3.69 + rank 5: 4.04 + rank 6: 3.98 + rank 7: 4.02 + rank 8: 3.92 + rank 9: 4.44 + rank 10: 3.84 + rank 11: 3.83 + rank 12: 3.94 + rank 13: 3.86 + rank 14: 3.97 + rank 15: 4.21 + rank 16: 4.46 + rank 17: 4.01 + rank 18: 4.07 + rank 19: 4.05 + rank 20: 4.03 + rank 21: 4.23 + rank 22: 4.60 + rank 23: 4.22 + rank 24: 3.94 + rank 25: 4.22 + rank 26: 3.94 + rank 27: 4.12 + rank 28: 4.42 + rank 29: 4.38 + rank 30: 4.42 + rank 31: 4.04 + rank 32: 4.13 + rank 33: 3.92 + rank 34: 4.01 + rank 35: 4.54 + rank 36: 4.21 + rank 37: 4.09 + rank 38: 4.49 + rank 39: 3.92 + rank 40: 3.75 + rank 41: 4.00 + rank 42: 3.91 + rank 43: 3.95 + rank 44: 3.96 + rank 45: 4.27 + rank 46: 3.77 + rank 47: 3.74 + rank 48: 4.11 + rank 49: 3.89 + rank 50: 4.96 + rank 51: 4.05 + rank 52: 4.19 + rank 53: 3.90 + rank 54: 3.92 + rank 55: 3.93 + rank 56: 3.77 + rank 57: 4.67 + rank 58: 3.88 + rank 59: 3.75 + rank 60: 3.77 + rank 61: 3.64 + rank 62: 3.93 + rank 63: 3.66 + optimizer: + rank 0: 55.88 + rank 1: 54.94 + rank 2: 54.50 + rank 3: 55.51 + rank 4: 54.44 + rank 5: 55.06 + rank 6: 55.32 + rank 7: 55.29 + rank 8: 54.97 + rank 9: 55.70 + rank 10: 54.63 + rank 11: 54.78 + rank 12: 54.98 + rank 13: 55.07 + rank 14: 55.01 + rank 15: 55.16 + rank 16: 56.26 + rank 17: 55.34 + rank 18: 55.49 + rank 19: 55.35 + rank 20: 55.29 + rank 21: 55.49 + rank 22: 56.59 + rank 23: 55.78 + rank 24: 54.69 + rank 25: 59.44 + rank 26: 56.50 + rank 27: 60.46 + rank 28: 64.87 + rank 29: 59.87 + rank 30: 59.20 + rank 31: 60.45 + rank 32: 55.24 + rank 33: 54.87 + rank 34: 55.04 + rank 35: 60.65 + rank 36: 55.42 + rank 37: 55.22 + rank 38: 55.62 + rank 39: 54.84 + rank 40: 54.66 + rank 41: 55.11 + rank 42: 55.04 + rank 43: 55.17 + rank 44: 54.79 + rank 45: 55.81 + rank 46: 54.75 + rank 47: 54.39 + rank 48: 55.15 + rank 49: 54.73 + rank 50: 67.31 + rank 51: 54.80 + rank 52: 55.35 + rank 53: 54.78 + rank 54: 54.96 + rank 55: 54.91 + rank 56: 54.43 + rank 57: 58.83 + rank 58: 54.81 + rank 59: 54.49 + rank 60: 54.82 + rank 61: 54.37 + rank 62: 54.85 + rank 63: 54.31 + interval-time: + rank 0: 0.99 + rank 1: 0.97 + rank 2: 1.01 + rank 3: 0.92 + rank 4: 0.99 + rank 5: 1.04 + rank 6: 0.93 + rank 7: 0.91 + rank 8: 0.76 + rank 9: 0.86 + rank 10: 0.86 + rank 11: 0.85 + rank 12: 0.85 + rank 13: 0.75 + rank 14: 0.75 + rank 15: 0.83 + rank 16: 0.84 + rank 17: 0.83 + rank 18: 0.80 + rank 19: 0.86 + rank 20: 0.73 + rank 21: 0.83 + rank 22: 0.74 + rank 23: 0.82 + rank 24: 0.95 + rank 25: 0.95 + rank 26: 0.90 + rank 27: 0.97 + rank 28: 0.92 + rank 29: 0.95 + rank 30: 0.99 + rank 31: 0.96 + rank 32: 0.90 + rank 33: 1.00 + rank 34: 0.92 + rank 35: 1.08 + rank 36: 1.20 + rank 37: 0.96 + rank 38: 0.88 + rank 39: 0.86 + rank 40: 0.83 + rank 41: 0.80 + rank 42: 0.83 + rank 43: 0.74 + rank 44: 0.73 + rank 45: 0.82 + rank 46: 0.74 + rank 47: 0.80 + rank 48: 0.92 + rank 49: 0.99 + rank 50: 0.86 + rank 51: 0.94 + rank 52: 0.81 + rank 53: 0.98 + rank 54: 0.92 + rank 55: 0.91 + rank 56: 0.84 + rank 57: 0.81 + rank 58: 0.82 + rank 59: 0.74 + rank 60: 0.73 + rank 61: 0.81 + rank 62: 0.72 + rank 63: 2.44 + [2026-10-16 19:56:33] iteration 2006/ 2384 | consumed samples: 64192 | elapsed time per iteration (ms): 220631.3 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 76041.8 | valid tokens/s: 76041.8 | physical tokens/s: 77329.8 | loss tokens/s: 56171.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.658300E-01 | lm-global-supervised-tokens: 1.239309E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.332965E-02 | lm-token-buffer-mib: 2.329936E+02 | global_load_balancing_loss: 9.987928E-01 | loss scale: 1.0 | grad norm: 0.273 | params norm: 11152.083 | params rms: 0.060385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:09:58.613906 | finish at 2026-10-17 19:06:31 + [2026-10-16 20:00:15] iteration 2007/ 2384 | consumed samples: 64224 | elapsed time per iteration (ms): 222440.7 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 75423.3 | valid tokens/s: 75423.3 | physical tokens/s: 75784.1 | loss tokens/s: 56817.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.635548E-01 | lm-global-supervised-tokens: 1.263862E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.411253E-02 | lm-token-buffer-mib: 2.331727E+02 | global_load_balancing_loss: 1.000536E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11152.055 | params rms: 0.060385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:17:40.129000 | finish at 2026-10-17 19:17:55 + [2026-10-16 20:04:04] iteration 2008/ 2384 | consumed samples: 64256 | elapsed time per iteration (ms): 228477.8 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 73430.3 | valid tokens/s: 73430.3 | physical tokens/s: 74370.2 | loss tokens/s: 55437.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.918694E-01 | lm-global-supervised-tokens: 1.266615E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.397353E-02 | lm-token-buffer-mib: 2.362801E+02 | global_load_balancing_loss: 9.996861E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11152.027 | params rms: 0.060385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:51:47.646484 | finish at 2026-10-17 19:55:51 + [2026-10-16 20:07:48] iteration 2009/ 2384 | consumed samples: 64288 | elapsed time per iteration (ms): 223733.5 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 74987.4 | valid tokens/s: 74987.4 | physical tokens/s: 75607.9 | loss tokens/s: 57171.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.869603E-01 | lm-global-supervised-tokens: 1.279107E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.356114E-02 | lm-token-buffer-mib: 2.346129E+02 | global_load_balancing_loss: 9.998631E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11152.000 | params rms: 0.060384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:18:20.077790 | finish at 2026-10-17 19:26:08 + [2026-10-16 20:11:52] iteration 2010/ 2384 | consumed samples: 64320 | elapsed time per iteration (ms): 244221.3 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.12% | FLOPs estimator: workload-aware | logical tokens/s: 68696.7 | valid tokens/s: 68696.7 | physical tokens/s: 69480.6 | loss tokens/s: 53134.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.039656E-01 | lm-global-supervised-tokens: 1.297666E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.515632E-02 | lm-token-buffer-mib: 2.355589E+02 | global_load_balancing_loss: 1.001548E+00 | loss scale: 1.0 | grad norm: 0.339 | params norm: 11151.971 | params rms: 0.060384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:22:18.777125 | finish at 2026-10-17 21:34:11 + [2026-10-16 20:15:32] iteration 2011/ 2384 | consumed samples: 64352 | elapsed time per iteration (ms): 219872.1 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 76304.3 | valid tokens/s: 76304.3 | physical tokens/s: 77091.5 | loss tokens/s: 61608.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.524560E-01 | lm-global-supervised-tokens: 1.354590E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.547320E-02 | lm-token-buffer-mib: 2.342681E+02 | global_load_balancing_loss: 1.002554E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11151.940 | params rms: 0.060384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:46:52.301524 | finish at 2026-10-17 19:02:24 + [2026-10-16 20:19:17] iteration 2012/ 2384 | consumed samples: 64384 | elapsed time per iteration (ms): 225683.5 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 74339.5 | valid tokens/s: 74339.5 | physical tokens/s: 75149.2 | loss tokens/s: 56449.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.433022E-01 | lm-global-supervised-tokens: 1.273971E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.261920E-02 | lm-token-buffer-mib: 2.329218E+02 | global_load_balancing_loss: 1.001466E+00 | loss scale: 1.0 | grad norm: 0.452 | params norm: 11151.909 | params rms: 0.060384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:19:14.269558 | finish at 2026-10-17 19:38:32 + [2026-10-16 20:23:01] iteration 2013/ 2384 | consumed samples: 64416 | elapsed time per iteration (ms): 223375.8 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 75107.5 | valid tokens/s: 75107.4 | physical tokens/s: 75672.9 | loss tokens/s: 57800.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.947007E-01 | lm-global-supervised-tokens: 1.291121E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.213246E-02 | lm-token-buffer-mib: 2.346796E+02 | global_load_balancing_loss: 1.000452E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11151.878 | params rms: 0.060384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:01:12.411412 | finish at 2026-10-17 19:24:13 + [2026-10-16 20:26:44] iteration 2014/ 2384 | consumed samples: 64448 | elapsed time per iteration (ms): 223303.1 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 75132.0 | valid tokens/s: 75132.0 | physical tokens/s: 75921.1 | loss tokens/s: 58348.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.843124E-01 | lm-global-supervised-tokens: 1.302938E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.329670E-02 | lm-token-buffer-mib: 2.352472E+02 | global_load_balancing_loss: 9.998767E-01 | loss scale: 1.0 | grad norm: 0.301 | params norm: 11151.850 | params rms: 0.060384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:57:02.154803 | finish at 2026-10-17 19:23:46 + [2026-10-16 20:30:28] iteration 2015/ 2384 | consumed samples: 64480 | elapsed time per iteration (ms): 224137.1 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.12% | FLOPs estimator: workload-aware | logical tokens/s: 74852.4 | valid tokens/s: 74852.4 | physical tokens/s: 75366.7 | loss tokens/s: 55221.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.000460E-01 | lm-global-supervised-tokens: 1.237725E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.106906E-02 | lm-token-buffer-mib: 2.325376E+02 | global_load_balancing_loss: 9.995071E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11151.822 | params rms: 0.060383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:58:26.598339 | finish at 2026-10-17 19:28:55 + [2026-10-16 20:34:10] iteration 2016/ 2384 | consumed samples: 64512 | elapsed time per iteration (ms): 221419.4 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 75771.2 | valid tokens/s: 75771.2 | physical tokens/s: 76120.7 | loss tokens/s: 52249.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.017988E-01 | lm-global-supervised-tokens: 1.156894E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.388206E-02 | lm-token-buffer-mib: 2.334011E+02 | global_load_balancing_loss: 9.986477E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11151.793 | params rms: 0.060383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:38:02.321117 | finish at 2026-10-17 19:12:12 + [2026-10-16 20:37:51] iteration 2017/ 2384 | consumed samples: 64544 | elapsed time per iteration (ms): 221912.3 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 75602.8 | valid tokens/s: 75602.8 | physical tokens/s: 76266.5 | loss tokens/s: 56835.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.275857E-01 | lm-global-supervised-tokens: 1.261256E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.287783E-02 | lm-token-buffer-mib: 2.337949E+02 | global_load_balancing_loss: 9.982844E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11151.760 | params rms: 0.060383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:37:21.819565 | finish at 2026-10-17 19:15:13 + [2026-10-16 20:41:33] iteration 2018/ 2384 | consumed samples: 64576 | elapsed time per iteration (ms): 221303.6 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 75810.8 | valid tokens/s: 75810.8 | physical tokens/s: 76489.0 | loss tokens/s: 58386.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.042849E-01 | lm-global-supervised-tokens: 1.292119E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.350210E-02 | lm-token-buffer-mib: 2.341765E+02 | global_load_balancing_loss: 9.991714E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11151.729 | params rms: 0.060383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:29:57.122077 | finish at 2026-10-17 19:11:30 + [2026-10-16 20:45:14] iteration 2019/ 2384 | consumed samples: 64608 | elapsed time per iteration (ms): 220961.5 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.15% | FLOPs estimator: workload-aware | logical tokens/s: 75928.1 | valid tokens/s: 75928.1 | physical tokens/s: 76534.2 | loss tokens/s: 57585.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.127025E-01 | lm-global-supervised-tokens: 1.272412E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.506165E-02 | lm-token-buffer-mib: 2.346299E+02 | global_load_balancing_loss: 1.001344E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11151.697 | params rms: 0.060383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:24:10.951651 | finish at 2026-10-17 19:09:25 + [2026-10-16 20:49:16] iteration 2020/ 2384 | consumed samples: 64640 | elapsed time per iteration (ms): 242167.0 | throughput per GPU (TFLOP/s/GPU): 17.5 | MFU 5.62% | FLOPs estimator: workload-aware | logical tokens/s: 69279.5 | valid tokens/s: 69279.5 | physical tokens/s: 70211.1 | loss tokens/s: 54701.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.329467E-01 | lm-global-supervised-tokens: 1.324684E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.349487E-02 | lm-token-buffer-mib: 2.356821E+02 | global_load_balancing_loss: 1.001128E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11151.666 | params rms: 0.060383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:29:08.795657 | finish at 2026-10-17 21:18:25 + [2026-10-16 20:52:56] iteration 2021/ 2384 | consumed samples: 64672 | elapsed time per iteration (ms): 220215.3 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 76185.3 | valid tokens/s: 76185.3 | physical tokens/s: 76621.5 | loss tokens/s: 58003.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.472034E-01 | lm-global-supervised-tokens: 1.277318E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.886706E-02 | lm-token-buffer-mib: 2.328904E+02 | global_load_balancing_loss: 1.000582E+00 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11151.633 | params rms: 0.060382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:12:18.143198 | finish at 2026-10-17 19:05:14 + [2026-10-16 20:56:41] iteration 2022/ 2384 | consumed samples: 64704 | elapsed time per iteration (ms): 224502.2 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 74730.7 | valid tokens/s: 74730.7 | physical tokens/s: 75496.4 | loss tokens/s: 56577.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.071619E-01 | lm-global-supervised-tokens: 1.270173E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.584049E-02 | lm-token-buffer-mib: 2.328270E+02 | global_load_balancing_loss: 1.002181E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11151.598 | params rms: 0.060382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:34:29.807062 | finish at 2026-10-17 19:31:10 + [2026-10-16 21:00:23] iteration 2023/ 2384 | consumed samples: 64736 | elapsed time per iteration (ms): 222510.1 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.29% | FLOPs estimator: workload-aware | logical tokens/s: 75399.7 | valid tokens/s: 75399.7 | physical tokens/s: 76082.3 | loss tokens/s: 58662.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.115098E-01 | lm-global-supervised-tokens: 1.305303E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.707444E-02 | lm-token-buffer-mib: 2.339582E+02 | global_load_balancing_loss: 1.001681E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11151.566 | params rms: 0.060382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:18:46.162068 | finish at 2026-10-17 19:19:09 + [2026-10-16 21:04:05] iteration 2024/ 2384 | consumed samples: 64768 | elapsed time per iteration (ms): 221750.7 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 75657.9 | valid tokens/s: 75657.9 | physical tokens/s: 76293.2 | loss tokens/s: 52930.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.132396E-01 | lm-global-supervised-tokens: 1.173737E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.306041E-02 | lm-token-buffer-mib: 2.339361E+02 | global_load_balancing_loss: 1.001736E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11151.535 | params rms: 0.060382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:10:30.255346 | finish at 2026-10-17 19:14:35 + [2026-10-16 21:07:45] iteration 2025/ 2384 | consumed samples: 64800 | elapsed time per iteration (ms): 220361.6 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 76134.8 | valid tokens/s: 76134.8 | physical tokens/s: 76904.9 | loss tokens/s: 55492.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.649534E-01 | lm-global-supervised-tokens: 1.222832E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.658630E-02 | lm-token-buffer-mib: 2.343761E+02 | global_load_balancing_loss: 9.991602E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11151.502 | params rms: 0.060382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:58:29.826184 | finish at 2026-10-17 19:06:15 + [2026-10-16 21:11:32] iteration 2026/ 2384 | consumed samples: 64832 | elapsed time per iteration (ms): 226407.2 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.66% | FLOPs estimator: workload-aware | logical tokens/s: 74101.7 | valid tokens/s: 74101.7 | physical tokens/s: 74315.9 | loss tokens/s: 52600.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.932001E-01 | lm-global-supervised-tokens: 1.190919E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.948985E-02 | lm-token-buffer-mib: 2.331127E+02 | global_load_balancing_loss: 9.982187E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11151.470 | params rms: 0.060382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:30:53.779684 | finish at 2026-10-17 19:42:25 + [2026-10-16 21:15:12] iteration 2027/ 2384 | consumed samples: 64864 | elapsed time per iteration (ms): 220004.6 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.60% | FLOPs estimator: workload-aware | logical tokens/s: 76258.3 | valid tokens/s: 76258.3 | physical tokens/s: 77282.2 | loss tokens/s: 56491.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.474447E-01 | lm-global-supervised-tokens: 1.242848E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.285291E-02 | lm-token-buffer-mib: 2.331876E+02 | global_load_balancing_loss: 9.987704E-01 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11151.437 | params rms: 0.060381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:49:01.626726 | finish at 2026-10-17 19:04:13 + [2026-10-16 21:18:54] iteration 2028/ 2384 | consumed samples: 64896 | elapsed time per iteration (ms): 222434.9 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 75425.2 | valid tokens/s: 75425.2 | physical tokens/s: 76205.6 | loss tokens/s: 58728.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.417309E-01 | lm-global-supervised-tokens: 1.306327E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.138757E-02 | lm-token-buffer-mib: 2.348121E+02 | global_load_balancing_loss: 9.995650E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11151.407 | params rms: 0.060381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:59:46.809053 | finish at 2026-10-17 19:18:41 + [2026-10-16 21:22:40] iteration 2029/ 2384 | consumed samples: 64928 | elapsed time per iteration (ms): 225800.4 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 74301.1 | valid tokens/s: 74301.1 | physical tokens/s: 74923.8 | loss tokens/s: 54351.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.116733E-01 | lm-global-supervised-tokens: 1.227254E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.609442E-02 | lm-token-buffer-mib: 2.347682E+02 | global_load_balancing_loss: 1.000701E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11151.376 | params rms: 0.060381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:15:59.126179 | finish at 2026-10-17 19:38:39 + [2026-10-16 21:26:48] iteration 2030/ 2384 | consumed samples: 64960 | elapsed time per iteration (ms): 247695.0 | throughput per GPU (TFLOP/s/GPU): 18.3 | MFU 5.85% | FLOPs estimator: workload-aware | logical tokens/s: 67733.3 | valid tokens/s: 67733.3 | physical tokens/s: 68753.2 | loss tokens/s: 52258.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.451287E-01 | lm-global-supervised-tokens: 1.294410E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.335961E-02 | lm-token-buffer-mib: 2.373069E+02 | global_load_balancing_loss: 1.001759E+00 | loss scale: 1.0 | grad norm: 0.180 | params norm: 11151.345 | params rms: 0.060381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:21:24.023984 | finish at 2026-10-17 21:48:12 + [2026-10-16 21:30:27] iteration 2031/ 2384 | consumed samples: 64992 | elapsed time per iteration (ms): 219480.1 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 76440.6 | valid tokens/s: 76440.6 | physical tokens/s: 76915.7 | loss tokens/s: 59774.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.957730E-01 | lm-global-supervised-tokens: 1.311930E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.681456E-02 | lm-token-buffer-mib: 2.326885E+02 | global_load_balancing_loss: 1.000590E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11151.312 | params rms: 0.060381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:31:16.486212 | finish at 2026-10-17 19:01:44 + [2026-10-16 21:34:14] iteration 2032/ 2384 | consumed samples: 65024 | elapsed time per iteration (ms): 226596.6 | throughput per GPU (TFLOP/s/GPU): 25.3 | MFU 8.11% | FLOPs estimator: workload-aware | logical tokens/s: 74039.9 | valid tokens/s: 74039.9 | physical tokens/s: 74351.5 | loss tokens/s: 59212.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.745246E-01 | lm-global-supervised-tokens: 1.341733E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.559575E-02 | lm-token-buffer-mib: 2.330109E+02 | global_load_balancing_loss: 1.002413E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11151.276 | params rms: 0.060381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:09:22.004059 | finish at 2026-10-17 19:43:36 + [2026-10-16 21:37:57] iteration 2033/ 2384 | consumed samples: 65056 | elapsed time per iteration (ms): 223501.7 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 75065.2 | valid tokens/s: 75065.2 | physical tokens/s: 76375.6 | loss tokens/s: 56380.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.255534E-01 | lm-global-supervised-tokens: 1.260106E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.866888E-02 | lm-token-buffer-mib: 2.366571E+02 | global_load_balancing_loss: 1.002511E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11151.245 | params rms: 0.060380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:47:29.094498 | finish at 2026-10-17 19:25:26 + [2026-10-16 21:41:38] iteration 2034/ 2384 | consumed samples: 65088 | elapsed time per iteration (ms): 220461.7 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 76100.2 | valid tokens/s: 76100.2 | physical tokens/s: 76773.9 | loss tokens/s: 58628.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.195698E-01 | lm-global-supervised-tokens: 1.292535E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.518681E-02 | lm-token-buffer-mib: 2.330536E+02 | global_load_balancing_loss: 9.993323E-01 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11151.215 | params rms: 0.060380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:26:01.586142 | finish at 2026-10-17 19:07:39 + [2026-10-16 21:45:20] iteration 2035/ 2384 | consumed samples: 65120 | elapsed time per iteration (ms): 222464.1 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 75415.4 | valid tokens/s: 75415.4 | physical tokens/s: 75840.0 | loss tokens/s: 53641.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.036257E-01 | lm-global-supervised-tokens: 1.193337E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.521851E-02 | lm-token-buffer-mib: 2.333511E+02 | global_load_balancing_loss: 9.987119E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11151.185 | params rms: 0.060380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:33:59.966949 | finish at 2026-10-17 19:19:20 + [2026-10-16 21:49:02] iteration 2036/ 2384 | consumed samples: 65152 | elapsed time per iteration (ms): 221688.2 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 75679.3 | valid tokens/s: 75679.3 | physical tokens/s: 76188.6 | loss tokens/s: 53710.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.484664E-01 | lm-global-supervised-tokens: 1.190700E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.276187E-02 | lm-token-buffer-mib: 2.326320E+02 | global_load_balancing_loss: 9.996180E-01 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11151.151 | params rms: 0.060380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:25:47.498992 | finish at 2026-10-17 19:14:49 + [2026-10-16 21:52:44] iteration 2037/ 2384 | consumed samples: 65184 | elapsed time per iteration (ms): 222260.8 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 75484.3 | valid tokens/s: 75484.3 | physical tokens/s: 76263.9 | loss tokens/s: 56061.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.566006E-01 | lm-global-supervised-tokens: 1.246024E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.603526E-02 | lm-token-buffer-mib: 2.330247E+02 | global_load_balancing_loss: 9.995747E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11151.118 | params rms: 0.060380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:25:24.493175 | finish at 2026-10-17 19:18:09 + [2026-10-16 21:56:27] iteration 2038/ 2384 | consumed samples: 65216 | elapsed time per iteration (ms): 223003.2 | throughput per GPU (TFLOP/s/GPU): 27.8 | MFU 8.89% | FLOPs estimator: workload-aware | logical tokens/s: 75232.8 | valid tokens/s: 75232.8 | physical tokens/s: 75558.5 | loss tokens/s: 59439.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.043201E-01 | lm-global-supervised-tokens: 1.325514E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.475038E-02 | lm-token-buffer-mib: 2.327056E+02 | global_load_balancing_loss: 9.998572E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11151.084 | params rms: 0.060379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:25:59.114726 | finish at 2026-10-17 19:22:26 + [2026-10-16 22:00:11] iteration 2039/ 2384 | consumed samples: 65248 | elapsed time per iteration (ms): 223965.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 74909.9 | valid tokens/s: 74909.9 | physical tokens/s: 75950.7 | loss tokens/s: 52975.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.029627E-01 | lm-global-supervised-tokens: 1.186456E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.430937E-02 | lm-token-buffer-mib: 2.359153E+02 | global_load_balancing_loss: 1.000035E+00 | loss scale: 1.0 | grad norm: 0.281 | params norm: 11151.049 | params rms: 0.060379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:27:47.952937 | finish at 2026-10-17 19:27:59 + [2026-10-16 22:04:15] iteration 2040/ 2384 | consumed samples: 65280 | elapsed time per iteration (ms): 244350.1 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 68660.5 | valid tokens/s: 68660.5 | physical tokens/s: 69209.8 | loss tokens/s: 51292.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.746301E-01 | lm-global-supervised-tokens: 1.253334E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.523080E-02 | lm-token-buffer-mib: 2.333687E+02 | global_load_balancing_loss: 1.001237E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11151.016 | params rms: 0.060379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:20:56.422276 | finish at 2026-10-17 21:25:12 +times across ranks (ms): + save-checkpoint: + rank 0: 24169.52 + rank 1: 24169.52 + rank 2: 24169.53 + rank 3: 24169.55 + rank 4: 24169.51 + rank 5: 24169.53 + rank 6: 24169.54 + rank 7: 24169.52 + rank 8: 24169.69 + rank 9: 24169.71 + rank 10: 24169.71 + rank 11: 24169.68 + rank 12: 24169.71 + rank 13: 24169.69 + rank 14: 24169.71 + rank 15: 24169.70 + rank 16: 24170.98 + rank 17: 24170.99 + rank 18: 24170.97 + rank 19: 24170.98 + rank 20: 24170.95 + rank 21: 24170.97 + rank 22: 24170.98 + rank 23: 24170.95 + rank 24: 24169.96 + rank 25: 24169.96 + rank 26: 24169.98 + rank 27: 24169.99 + rank 28: 24169.98 + rank 29: 24170.00 + rank 30: 24169.98 + rank 31: 24170.05 + rank 32: 24168.79 + rank 33: 24168.80 + rank 34: 24168.78 + rank 35: 24168.79 + rank 36: 24168.76 + rank 37: 24168.80 + rank 38: 24168.76 + rank 39: 24168.79 + rank 40: 24168.94 + rank 41: 24168.97 + rank 42: 24168.93 + rank 43: 24168.93 + rank 44: 24168.94 + rank 45: 24168.95 + rank 46: 24168.96 + rank 47: 24168.96 + rank 48: 24169.10 + rank 49: 24169.14 + rank 50: 24169.13 + rank 51: 24169.09 + rank 52: 24169.09 + rank 53: 24169.12 + rank 54: 24169.09 + rank 55: 24169.14 + rank 56: 24169.82 + rank 57: 24169.81 + rank 58: 24169.81 + rank 59: 24169.81 + rank 60: 24169.86 + rank 61: 24169.81 + rank 62: 24169.79 + rank 63: 24169.81 + [2026-10-16 22:08:22] iteration 2041/ 2384 | consumed samples: 65312 | elapsed time per iteration (ms): 221715.4 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 75670.0 | valid tokens/s: 75670.0 | physical tokens/s: 76438.8 | loss tokens/s: 58264.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.395319E-01 | lm-global-supervised-tokens: 1.291806E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.575032E-02 | lm-token-buffer-mib: 2.337086E+02 | global_load_balancing_loss: 1.000841E+00 | loss scale: 1.0 | grad norm: 0.236 | params norm: 11150.982 | params rms: 0.060379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:07:28.378350 | finish at 2026-10-17 19:15:50 + [2026-10-16 22:12:09] iteration 2042/ 2384 | consumed samples: 65344 | elapsed time per iteration (ms): 226909.8 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 73937.8 | valid tokens/s: 73937.8 | physical tokens/s: 74648.3 | loss tokens/s: 59211.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.847852E-01 | lm-global-supervised-tokens: 1.343568E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.546453E-02 | lm-token-buffer-mib: 2.327021E+02 | global_load_balancing_loss: 1.000122E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11150.947 | params rms: 0.060379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:33:23.148846 | finish at 2026-10-17 19:45:32 + [2026-10-16 22:15:49] iteration 2043/ 2384 | consumed samples: 65376 | elapsed time per iteration (ms): 220714.5 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.05% | FLOPs estimator: workload-aware | logical tokens/s: 76013.1 | valid tokens/s: 76013.1 | physical tokens/s: 76627.1 | loss tokens/s: 59131.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.727199E-01 | lm-global-supervised-tokens: 1.305119E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.771234E-02 | lm-token-buffer-mib: 2.336344E+02 | global_load_balancing_loss: 1.001357E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11150.912 | params rms: 0.060379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:54:23.645947 | finish at 2026-10-17 19:10:13 + [2026-10-16 22:19:35] iteration 2044/ 2384 | consumed samples: 65408 | elapsed time per iteration (ms): 225465.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 74411.2 | valid tokens/s: 74411.2 | physical tokens/s: 75857.4 | loss tokens/s: 54983.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.199663E-01 | lm-global-supervised-tokens: 1.239684E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.118598E-02 | lm-token-buffer-mib: 2.375754E+02 | global_load_balancing_loss: 1.001518E+00 | loss scale: 1.0 | grad norm: 0.371 | params norm: 11150.880 | params rms: 0.060378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:17:38.342834 | finish at 2026-10-17 19:37:13 + [2026-10-16 22:23:16] iteration 2045/ 2384 | consumed samples: 65440 | elapsed time per iteration (ms): 221147.0 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.53% | FLOPs estimator: workload-aware | logical tokens/s: 75864.4 | valid tokens/s: 75864.4 | physical tokens/s: 76313.7 | loss tokens/s: 55438.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.104108E-01 | lm-global-supervised-tokens: 1.226016E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.938587E-02 | lm-token-buffer-mib: 2.336009E+02 | global_load_balancing_loss: 1.001897E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11150.846 | params rms: 0.060378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:49:28.849028 | finish at 2026-10-17 19:12:45 + [2026-10-16 22:26:57] iteration 2046/ 2384 | consumed samples: 65472 | elapsed time per iteration (ms): 220721.2 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 76010.8 | valid tokens/s: 76010.7 | physical tokens/s: 76672.6 | loss tokens/s: 60551.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.604456E-01 | lm-global-supervised-tokens: 1.336492E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.300356E-02 | lm-token-buffer-mib: 2.338268E+02 | global_load_balancing_loss: 1.001051E+00 | loss scale: 1.0 | grad norm: 0.268 | params norm: 11150.808 | params rms: 0.060378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:43:23.759552 | finish at 2026-10-17 19:10:20 + [2026-10-16 22:30:37] iteration 2047/ 2384 | consumed samples: 65504 | elapsed time per iteration (ms): 220244.9 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 76175.3 | valid tokens/s: 76175.3 | physical tokens/s: 77105.7 | loss tokens/s: 56097.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.116555E-01 | lm-global-supervised-tokens: 1.235507E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.642164E-02 | lm-token-buffer-mib: 2.351888E+02 | global_load_balancing_loss: 9.994797E-01 | loss scale: 1.0 | grad norm: 0.262 | params norm: 11150.774 | params rms: 0.060378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:37:02.527841 | finish at 2026-10-17 19:07:39 + [2026-10-16 22:34:21] iteration 2048/ 2384 | consumed samples: 65536 | elapsed time per iteration (ms): 224405.4 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 74762.9 | valid tokens/s: 74762.9 | physical tokens/s: 75497.6 | loss tokens/s: 60931.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.656521E-01 | lm-global-supervised-tokens: 1.367344E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.448582E-02 | lm-token-buffer-mib: 2.343132E+02 | global_load_balancing_loss: 9.984642E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11150.743 | params rms: 0.060378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:56:40.226910 | finish at 2026-10-17 19:31:01 + [2026-10-16 22:38:04] iteration 2049/ 2384 | consumed samples: 65568 | elapsed time per iteration (ms): 223260.8 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.25% | FLOPs estimator: workload-aware | logical tokens/s: 75146.2 | valid tokens/s: 75146.2 | physical tokens/s: 76016.9 | loss tokens/s: 53176.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.748905E-01 | lm-global-supervised-tokens: 1.187221E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.350729E-02 | lm-token-buffer-mib: 2.331474E+02 | global_load_balancing_loss: 1.000283E+00 | loss scale: 1.0 | grad norm: 0.252 | params norm: 11150.712 | params rms: 0.060377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:46:32.358617 | finish at 2026-10-17 19:24:37 + [2026-10-16 22:42:06] iteration 2050/ 2384 | consumed samples: 65600 | elapsed time per iteration (ms): 241650.9 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 69427.5 | valid tokens/s: 69427.5 | physical tokens/s: 69697.4 | loss tokens/s: 52111.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.807343E-01 | lm-global-supervised-tokens: 1.259275E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.961928E-02 | lm-token-buffer-mib: 2.331634E+02 | global_load_balancing_loss: 9.980163E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11150.677 | params rms: 0.060377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:25:11.396342 | finish at 2026-10-17 21:07:18 + [2026-10-16 22:45:53] iteration 2051/ 2384 | consumed samples: 65632 | elapsed time per iteration (ms): 227211.3 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 73839.6 | valid tokens/s: 73839.6 | physical tokens/s: 74735.4 | loss tokens/s: 53367.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.602360E-01 | lm-global-supervised-tokens: 1.212573E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.910051E-02 | lm-token-buffer-mib: 2.373548E+02 | global_load_balancing_loss: 1.001676E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11150.642 | params rms: 0.060377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:01:01.365724 | finish at 2026-10-17 19:46:55 + [2026-10-16 22:49:38] iteration 2052/ 2384 | consumed samples: 65664 | elapsed time per iteration (ms): 224338.2 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.31% | FLOPs estimator: workload-aware | logical tokens/s: 74785.1 | valid tokens/s: 74785.1 | physical tokens/s: 75566.2 | loss tokens/s: 54731.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.250310E-01 | lm-global-supervised-tokens: 1.227830E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.382964E-02 | lm-token-buffer-mib: 2.341517E+02 | global_load_balancing_loss: 1.000450E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11150.609 | params rms: 0.060377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:41:20.266916 | finish at 2026-10-17 19:30:58 + [2026-10-16 22:53:22] iteration 2053/ 2384 | consumed samples: 65696 | elapsed time per iteration (ms): 224209.0 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 74828.3 | valid tokens/s: 74828.3 | physical tokens/s: 75496.7 | loss tokens/s: 53767.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.166589E-01 | lm-global-supervised-tokens: 1.205520E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.696217E-02 | lm-token-buffer-mib: 2.337049E+02 | global_load_balancing_loss: 1.001004E+00 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11150.583 | params rms: 0.060377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:36:53.191190 | finish at 2026-10-17 19:30:15 + [2026-10-16 22:57:05] iteration 2054/ 2384 | consumed samples: 65728 | elapsed time per iteration (ms): 223152.3 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 75182.6 | valid tokens/s: 75182.6 | physical tokens/s: 76091.1 | loss tokens/s: 57536.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.411821E-01 | lm-global-supervised-tokens: 1.283946E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.001098E-02 | lm-token-buffer-mib: 2.340642E+02 | global_load_balancing_loss: 1.000234E+00 | loss scale: 1.0 | grad norm: 0.214 | params norm: 11150.558 | params rms: 0.060377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:27:20.260456 | finish at 2026-10-17 19:24:25 + [2026-10-16 23:00:46] iteration 2055/ 2384 | consumed samples: 65760 | elapsed time per iteration (ms): 221430.8 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 75767.2 | valid tokens/s: 75767.2 | physical tokens/s: 76143.6 | loss tokens/s: 57250.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.969335E-01 | lm-global-supervised-tokens: 1.267710E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.598879E-02 | lm-token-buffer-mib: 2.334420E+02 | global_load_balancing_loss: 9.999035E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11150.531 | params rms: 0.060377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:14:10.725814 | finish at 2026-10-17 19:14:57 + [2026-10-16 23:04:29] iteration 2056/ 2384 | consumed samples: 65792 | elapsed time per iteration (ms): 222474.2 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 75411.9 | valid tokens/s: 75411.9 | physical tokens/s: 75970.6 | loss tokens/s: 55640.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.420437E-01 | lm-global-supervised-tokens: 1.237846E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.274472E-02 | lm-token-buffer-mib: 2.331600E+02 | global_load_balancing_loss: 9.996421E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11150.504 | params rms: 0.060376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:16:11.526030 | finish at 2026-10-17 19:20:40 + [2026-10-16 23:08:10] iteration 2057/ 2384 | consumed samples: 65824 | elapsed time per iteration (ms): 221458.0 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75757.9 | valid tokens/s: 75757.9 | physical tokens/s: 76176.5 | loss tokens/s: 56289.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.655000E-01 | lm-global-supervised-tokens: 1.246574E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.275928E-02 | lm-token-buffer-mib: 2.337400E+02 | global_load_balancing_loss: 1.000434E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11150.474 | params rms: 0.060376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:06:56.779547 | finish at 2026-10-17 19:15:07 + [2026-10-16 23:11:55] iteration 2058/ 2384 | consumed samples: 65856 | elapsed time per iteration (ms): 224699.0 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.86% | FLOPs estimator: workload-aware | logical tokens/s: 74665.0 | valid tokens/s: 74665.0 | physical tokens/s: 75743.3 | loss tokens/s: 62918.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.762482E-01 | lm-global-supervised-tokens: 1.413779E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.007199E-02 | lm-token-buffer-mib: 2.370211E+02 | global_load_balancing_loss: 9.997505E-01 | loss scale: 1.0 | grad norm: 0.278 | params norm: 11150.445 | params rms: 0.060376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:20:51.889740 | finish at 2026-10-17 19:32:47 + [2026-10-16 23:15:38] iteration 2059/ 2384 | consumed samples: 65888 | elapsed time per iteration (ms): 222898.8 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 75268.2 | valid tokens/s: 75268.2 | physical tokens/s: 75670.3 | loss tokens/s: 54676.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.137384E-01 | lm-global-supervised-tokens: 1.218726E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.375900E-02 | lm-token-buffer-mib: 2.328274E+02 | global_load_balancing_loss: 1.000980E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11150.421 | params rms: 0.060376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:07:22.124689 | finish at 2026-10-17 19:23:00 + [2026-10-16 23:19:40] iteration 2060/ 2384 | consumed samples: 65920 | elapsed time per iteration (ms): 242431.3 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 5.99% | FLOPs estimator: workload-aware | logical tokens/s: 69204.0 | valid tokens/s: 69204.0 | physical tokens/s: 69881.9 | loss tokens/s: 50367.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.317966E-01 | lm-global-supervised-tokens: 1.221064E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.285345E-02 | lm-token-buffer-mib: 2.357574E+02 | global_load_balancing_loss: 1.000528E+00 | loss scale: 1.0 | grad norm: 0.333 | params norm: 11150.397 | params rms: 0.060376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:49:07.741253 | finish at 2026-10-17 21:08:48 + [2026-10-16 23:23:23] iteration 2061/ 2384 | consumed samples: 65952 | elapsed time per iteration (ms): 223062.5 | throughput per GPU (TFLOP/s/GPU): 26.1 | MFU 8.36% | FLOPs estimator: workload-aware | logical tokens/s: 75212.8 | valid tokens/s: 75212.8 | physical tokens/s: 75871.8 | loss tokens/s: 58310.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.032173E-01 | lm-global-supervised-tokens: 1.300684E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.012975E-02 | lm-token-buffer-mib: 2.347470E+02 | global_load_balancing_loss: 9.990149E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11150.372 | params rms: 0.060376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:00:49.191427 | finish at 2026-10-17 19:24:13 + [2026-10-16 23:27:06] iteration 2062/ 2384 | consumed samples: 65984 | elapsed time per iteration (ms): 222355.6 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 75452.0 | valid tokens/s: 75452.0 | physical tokens/s: 75963.6 | loss tokens/s: 58026.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.972634E-01 | lm-global-supervised-tokens: 1.290244E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.324283E-02 | lm-token-buffer-mib: 2.333122E+02 | global_load_balancing_loss: 1.000681E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11150.344 | params rms: 0.060375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:53:18.510685 | finish at 2026-10-17 19:20:24 + [2026-10-16 23:30:48] iteration 2063/ 2384 | consumed samples: 66016 | elapsed time per iteration (ms): 222304.6 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 75469.3 | valid tokens/s: 75469.3 | physical tokens/s: 76422.4 | loss tokens/s: 59718.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.415769E-01 | lm-global-supervised-tokens: 1.327578E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.484304E-02 | lm-token-buffer-mib: 2.342094E+02 | global_load_balancing_loss: 1.001551E+00 | loss scale: 1.0 | grad norm: 0.314 | params norm: 11150.314 | params rms: 0.060375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:49:19.789993 | finish at 2026-10-17 19:20:08 + [2026-10-16 23:34:29] iteration 2064/ 2384 | consumed samples: 66048 | elapsed time per iteration (ms): 220701.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 76017.6 | valid tokens/s: 76017.6 | physical tokens/s: 76825.1 | loss tokens/s: 60306.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.351832E-01 | lm-global-supervised-tokens: 1.330966E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.781700E-02 | lm-token-buffer-mib: 2.351714E+02 | global_load_balancing_loss: 9.997619E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11150.285 | params rms: 0.060375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:37:04.388046 | finish at 2026-10-17 19:11:33 + [2026-10-16 23:38:13] iteration 2065/ 2384 | consumed samples: 66080 | elapsed time per iteration (ms): 224192.2 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 74833.9 | valid tokens/s: 74833.9 | physical tokens/s: 75440.7 | loss tokens/s: 57178.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.724974E-01 | lm-global-supervised-tokens: 1.281904E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.740205E-02 | lm-token-buffer-mib: 2.337397E+02 | global_load_balancing_loss: 1.000484E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11150.254 | params rms: 0.060375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:51:57.318856 | finish at 2026-10-17 19:30:10 + [2026-10-16 23:41:55] iteration 2066/ 2384 | consumed samples: 66112 | elapsed time per iteration (ms): 222051.1 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 75555.6 | valid tokens/s: 75555.6 | physical tokens/s: 76233.6 | loss tokens/s: 57804.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.545056E-01 | lm-global-supervised-tokens: 1.283556E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.817494E-02 | lm-token-buffer-mib: 2.344322E+02 | global_load_balancing_loss: 1.001682E+00 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11150.222 | params rms: 0.060375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:36:52.236992 | finish at 2026-10-17 19:18:47 + [2026-10-16 23:45:43] iteration 2067/ 2384 | consumed samples: 66144 | elapsed time per iteration (ms): 228142.6 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 73538.2 | valid tokens/s: 73538.2 | physical tokens/s: 74425.0 | loss tokens/s: 57458.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.185504E-01 | lm-global-supervised-tokens: 1.310872E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.624960E-02 | lm-token-buffer-mib: 2.354453E+02 | global_load_balancing_loss: 1.000994E+00 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11150.191 | params rms: 0.060375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:05:21.210190 | finish at 2026-10-17 19:51:04 + [2026-10-16 23:49:30] iteration 2068/ 2384 | consumed samples: 66176 | elapsed time per iteration (ms): 226291.1 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 74139.8 | valid tokens/s: 74139.8 | physical tokens/s: 75553.0 | loss tokens/s: 58561.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.816686E-01 | lm-global-supervised-tokens: 1.325186E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.315064E-02 | lm-token-buffer-mib: 2.365169E+02 | global_load_balancing_loss: 1.001976E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11150.161 | params rms: 0.060374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:51:47.979170 | finish at 2026-10-17 19:41:18 + [2026-10-16 23:53:12] iteration 2069/ 2384 | consumed samples: 66208 | elapsed time per iteration (ms): 222852.5 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 75283.9 | valid tokens/s: 75283.9 | physical tokens/s: 75841.1 | loss tokens/s: 57727.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.188166E-01 | lm-global-supervised-tokens: 1.286467E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.807077E-02 | lm-token-buffer-mib: 2.334690E+02 | global_load_balancing_loss: 1.000499E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11150.130 | params rms: 0.060374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:29:58.547627 | finish at 2026-10-17 19:23:11 + [2026-10-16 23:57:15] iteration 2070/ 2384 | consumed samples: 66240 | elapsed time per iteration (ms): 242935.6 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 69060.3 | valid tokens/s: 69060.3 | physical tokens/s: 69390.2 | loss tokens/s: 51757.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.310875E-01 | lm-global-supervised-tokens: 1.257371E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.928383E-02 | lm-token-buffer-mib: 2.332070E+02 | global_load_balancing_loss: 9.982639E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11150.098 | params rms: 0.060374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:11:21.779836 | finish at 2026-10-17 21:08:37 + [2026-10-17 00:00:58] iteration 2071/ 2384 | consumed samples: 66272 | elapsed time per iteration (ms): 222969.4 | throughput per GPU (TFLOP/s/GPU): 26.7 | MFU 8.56% | FLOPs estimator: workload-aware | logical tokens/s: 75244.3 | valid tokens/s: 75244.3 | physical tokens/s: 75619.6 | loss tokens/s: 57317.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.639652E-01 | lm-global-supervised-tokens: 1.278009E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.255852E-02 | lm-token-buffer-mib: 2.328786E+02 | global_load_balancing_loss: 9.999123E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11150.066 | params rms: 0.060374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:23:09.407850 | finish at 2026-10-17 19:24:08 + [2026-10-17 00:04:42] iteration 2072/ 2384 | consumed samples: 66304 | elapsed time per iteration (ms): 223634.0 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75020.7 | valid tokens/s: 75020.7 | physical tokens/s: 75742.3 | loss tokens/s: 60570.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.196500E-01 | lm-global-supervised-tokens: 1.354558E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.933094E-02 | lm-token-buffer-mib: 2.343762E+02 | global_load_balancing_loss: 9.992463E-01 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11150.035 | params rms: 0.060374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:22:53.798647 | finish at 2026-10-17 19:27:36 + [2026-10-17 00:08:26] iteration 2073/ 2384 | consumed samples: 66336 | elapsed time per iteration (ms): 224019.2 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 74891.7 | valid tokens/s: 74891.7 | physical tokens/s: 75332.1 | loss tokens/s: 52562.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.974453E-01 | lm-global-supervised-tokens: 1.177502E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.576853E-02 | lm-token-buffer-mib: 2.332092E+02 | global_load_balancing_loss: 1.000232E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11150.003 | params rms: 0.060374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:21:09.977752 | finish at 2026-10-17 19:29:36 + [2026-10-17 00:12:10] iteration 2074/ 2384 | consumed samples: 66368 | elapsed time per iteration (ms): 223808.0 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 74962.4 | valid tokens/s: 74962.4 | physical tokens/s: 75774.1 | loss tokens/s: 61314.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.468934E-01 | lm-global-supervised-tokens: 1.372260E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.067517E-02 | lm-token-buffer-mib: 2.359700E+02 | global_load_balancing_loss: 1.000688E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11149.963 | params rms: 0.060373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:16:20.493479 | finish at 2026-10-17 19:28:30 + [2026-10-17 00:15:50] iteration 2075/ 2384 | consumed samples: 66400 | elapsed time per iteration (ms): 220184.8 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 76195.8 | valid tokens/s: 76195.8 | physical tokens/s: 77198.6 | loss tokens/s: 60081.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.117234E-01 | lm-global-supervised-tokens: 1.322912E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.177086E-02 | lm-token-buffer-mib: 2.344694E+02 | global_load_balancing_loss: 1.001085E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11149.928 | params rms: 0.060373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:53:57.096468 | finish at 2026-10-17 19:09:47 + [2026-10-17 00:19:30] iteration 2076/ 2384 | consumed samples: 66432 | elapsed time per iteration (ms): 219677.9 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.45% | FLOPs estimator: workload-aware | logical tokens/s: 76371.8 | valid tokens/s: 76371.8 | physical tokens/s: 76896.3 | loss tokens/s: 57768.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.735673E-01 | lm-global-supervised-tokens: 1.269039E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.696281E-02 | lm-token-buffer-mib: 2.336260E+02 | global_load_balancing_loss: 1.001694E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11149.896 | params rms: 0.060373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:47:40.791534 | finish at 2026-10-17 19:07:10 + [2026-10-17 00:23:12] iteration 2077/ 2384 | consumed samples: 66464 | elapsed time per iteration (ms): 222060.6 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75552.4 | valid tokens/s: 75552.4 | physical tokens/s: 76421.4 | loss tokens/s: 60139.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.557148E-01 | lm-global-supervised-tokens: 1.335468E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.843280E-02 | lm-token-buffer-mib: 2.340334E+02 | global_load_balancing_loss: 1.001496E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11149.865 | params rms: 0.060373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:56:12.599529 | finish at 2026-10-17 19:19:24 + [2026-10-17 00:26:54] iteration 2078/ 2384 | consumed samples: 66496 | elapsed time per iteration (ms): 222432.3 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 75425.9 | valid tokens/s: 75425.9 | physical tokens/s: 76161.3 | loss tokens/s: 59534.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.272400E-01 | lm-global-supervised-tokens: 1.324243E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.185866E-02 | lm-token-buffer-mib: 2.331470E+02 | global_load_balancing_loss: 1.000704E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11149.831 | params rms: 0.060373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:54:24.295574 | finish at 2026-10-17 19:21:18 + [2026-10-17 00:30:35] iteration 2079/ 2384 | consumed samples: 66528 | elapsed time per iteration (ms): 220551.5 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 76069.3 | valid tokens/s: 76069.3 | physical tokens/s: 77829.6 | loss tokens/s: 55601.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.880089E-01 | lm-global-supervised-tokens: 1.226299E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.745198E-02 | lm-token-buffer-mib: 2.343372E+02 | global_load_balancing_loss: 1.001284E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11149.798 | params rms: 0.060373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:41:08.203089 | finish at 2026-10-17 19:11:43 + [2026-10-17 00:34:42] iteration 2080/ 2384 | consumed samples: 66560 | elapsed time per iteration (ms): 247092.9 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 67898.3 | valid tokens/s: 67898.3 | physical tokens/s: 68504.0 | loss tokens/s: 51939.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.825091E-01 | lm-global-supervised-tokens: 1.283378E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.350239E-02 | lm-token-buffer-mib: 2.343970E+02 | global_load_balancing_loss: 9.992663E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11149.769 | params rms: 0.060372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:51:56.237335 | finish at 2026-10-17 21:26:38 +times across ranks (ms): + save-checkpoint: + rank 0: 24235.24 + rank 1: 24235.23 + rank 2: 24235.28 + rank 3: 24235.24 + rank 4: 24235.24 + rank 5: 24235.25 + rank 6: 24235.26 + rank 7: 24235.27 + rank 8: 24233.71 + rank 9: 24233.70 + rank 10: 24233.72 + rank 11: 24233.71 + rank 12: 24233.68 + rank 13: 24233.71 + rank 14: 24233.68 + rank 15: 24233.73 + rank 16: 24233.71 + rank 17: 24233.70 + rank 18: 24233.69 + rank 19: 24233.67 + rank 20: 24233.68 + rank 21: 24233.68 + rank 22: 24233.69 + rank 23: 24233.69 + rank 24: 24233.77 + rank 25: 24233.78 + rank 26: 24233.94 + rank 27: 24233.78 + rank 28: 24233.80 + rank 29: 24233.80 + rank 30: 24233.78 + rank 31: 24233.79 + rank 32: 24233.67 + rank 33: 24233.68 + rank 34: 24233.68 + rank 35: 24233.69 + rank 36: 24233.64 + rank 37: 24233.67 + rank 38: 24233.65 + rank 39: 24233.68 + rank 40: 24233.58 + rank 41: 24233.55 + rank 42: 24233.54 + rank 43: 24233.58 + rank 44: 24233.61 + rank 45: 24233.55 + rank 46: 24233.54 + rank 47: 24233.58 + rank 48: 24233.41 + rank 49: 24233.44 + rank 50: 24233.45 + rank 51: 24233.38 + rank 52: 24233.40 + rank 53: 24233.45 + rank 54: 24233.39 + rank 55: 24233.39 + rank 56: 24233.82 + rank 57: 24233.88 + rank 58: 24233.82 + rank 59: 24233.82 + rank 60: 24233.82 + rank 61: 24233.80 + rank 62: 24233.81 + rank 63: 24233.82 + [2026-10-17 00:38:59] iteration 2081/ 2384 | consumed samples: 66592 | elapsed time per iteration (ms): 232384.7 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.99% | FLOPs estimator: workload-aware | logical tokens/s: 72195.7 | valid tokens/s: 72195.7 | physical tokens/s: 72840.6 | loss tokens/s: 55846.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.609133E-01 | lm-global-supervised-tokens: 1.297789E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.353585E+00 | lm-token-buffer-mib: 2.355309E+02 | global_load_balancing_loss: 9.992558E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11149.744 | params rms: 0.060372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:33:32.559350 | finish at 2026-10-17 20:12:31 + [2026-10-17 00:42:44] iteration 2082/ 2384 | consumed samples: 66624 | elapsed time per iteration (ms): 225800.6 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 74300.9 | valid tokens/s: 74300.9 | physical tokens/s: 74870.2 | loss tokens/s: 56722.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.730042E-01 | lm-global-supervised-tokens: 1.280804E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.296534E-02 | lm-token-buffer-mib: 2.336937E+02 | global_load_balancing_loss: 1.000124E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11149.716 | params rms: 0.060372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:56:31.776896 | finish at 2026-10-17 19:39:16 + [2026-10-17 00:46:30] iteration 2083/ 2384 | consumed samples: 66656 | elapsed time per iteration (ms): 225118.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 74525.9 | valid tokens/s: 74525.9 | physical tokens/s: 76068.9 | loss tokens/s: 52772.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.400972E-01 | lm-global-supervised-tokens: 1.188006E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.459452E-02 | lm-token-buffer-mib: 2.382769E+02 | global_load_balancing_loss: 1.001501E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11149.689 | params rms: 0.060372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:49:20.737679 | finish at 2026-10-17 19:35:50 + [2026-10-17 00:50:13] iteration 2084/ 2384 | consumed samples: 66688 | elapsed time per iteration (ms): 223011.3 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 75230.3 | valid tokens/s: 75230.3 | physical tokens/s: 76046.5 | loss tokens/s: 57171.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.324401E-01 | lm-global-supervised-tokens: 1.274977E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.056557E-02 | lm-token-buffer-mib: 2.337904E+02 | global_load_balancing_loss: 1.000240E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11149.660 | params rms: 0.060372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:35:03.397322 | finish at 2026-10-17 19:25:16 + [2026-10-17 00:53:58] iteration 2085/ 2384 | consumed samples: 66720 | elapsed time per iteration (ms): 225517.2 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 74394.3 | valid tokens/s: 74394.3 | physical tokens/s: 75878.4 | loss tokens/s: 58112.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.593706E-01 | lm-global-supervised-tokens: 1.310531E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.346146E-02 | lm-token-buffer-mib: 2.373569E+02 | global_load_balancing_loss: 9.996004E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11149.633 | params rms: 0.060372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:43:49.633531 | finish at 2026-10-17 19:37:48 + [2026-10-17 00:57:43] iteration 2086/ 2384 | consumed samples: 66752 | elapsed time per iteration (ms): 224604.1 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.13% | FLOPs estimator: workload-aware | logical tokens/s: 74696.8 | valid tokens/s: 74696.8 | physical tokens/s: 75969.4 | loss tokens/s: 58471.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.586580E-01 | lm-global-supervised-tokens: 1.313291E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.005592E-02 | lm-token-buffer-mib: 2.374013E+02 | global_load_balancing_loss: 1.000106E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11149.607 | params rms: 0.060371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:35:32.024212 | finish at 2026-10-17 19:33:15 + [2026-10-17 01:01:23] iteration 2087/ 2384 | consumed samples: 66784 | elapsed time per iteration (ms): 220218.9 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 76184.1 | valid tokens/s: 76184.1 | physical tokens/s: 76827.1 | loss tokens/s: 58909.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.529522E-01 | lm-global-supervised-tokens: 1.297295E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.504236E-02 | lm-token-buffer-mib: 2.342586E+02 | global_load_balancing_loss: 1.000902E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11149.581 | params rms: 0.060371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:10:05.005925 | finish at 2026-10-17 19:11:28 + [2026-10-17 01:05:07] iteration 2088/ 2384 | consumed samples: 66816 | elapsed time per iteration (ms): 223916.3 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 74926.2 | valid tokens/s: 74926.2 | physical tokens/s: 75594.0 | loss tokens/s: 56362.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.926745E-01 | lm-global-supervised-tokens: 1.262044E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.255804E-02 | lm-token-buffer-mib: 2.326982E+02 | global_load_balancing_loss: 1.000064E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11149.553 | params rms: 0.060371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:24:39.219807 | finish at 2026-10-17 19:29:46 + [2026-10-17 01:08:49] iteration 2089/ 2384 | consumed samples: 66848 | elapsed time per iteration (ms): 222129.9 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 75528.9 | valid tokens/s: 75528.9 | physical tokens/s: 75978.9 | loss tokens/s: 56324.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.360655E-01 | lm-global-supervised-tokens: 1.251138E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.365280E-02 | lm-token-buffer-mib: 2.328980E+02 | global_load_balancing_loss: 1.000445E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11149.521 | params rms: 0.060371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:12:08.320705 | finish at 2026-10-17 19:20:57 + [2026-10-17 01:12:54] iteration 2090/ 2384 | consumed samples: 66880 | elapsed time per iteration (ms): 245118.7 | throughput per GPU (TFLOP/s/GPU): 17.6 | MFU 5.63% | FLOPs estimator: workload-aware | logical tokens/s: 68445.2 | valid tokens/s: 68445.2 | physical tokens/s: 69292.8 | loss tokens/s: 51473.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.584202E-01 | lm-global-supervised-tokens: 1.261714E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.454566E-02 | lm-token-buffer-mib: 2.344917E+02 | global_load_balancing_loss: 9.996715E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11149.488 | params rms: 0.060371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:01:04.891850 | finish at 2026-10-17 21:13:59 + [2026-10-17 01:16:38] iteration 2091/ 2384 | consumed samples: 66912 | elapsed time per iteration (ms): 223800.1 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 74965.1 | valid tokens/s: 74965.1 | physical tokens/s: 75924.0 | loss tokens/s: 55262.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.034779E-01 | lm-global-supervised-tokens: 1.236777E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.848736E-02 | lm-token-buffer-mib: 2.361853E+02 | global_load_balancing_loss: 1.000998E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11149.459 | params rms: 0.060371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:12:53.433867 | finish at 2026-10-17 19:29:31 + [2026-10-17 01:20:22] iteration 2092/ 2384 | consumed samples: 66944 | elapsed time per iteration (ms): 224386.4 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 74769.2 | valid tokens/s: 74769.2 | physical tokens/s: 75318.4 | loss tokens/s: 56547.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.963782E-01 | lm-global-supervised-tokens: 1.268855E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.251424E-02 | lm-token-buffer-mib: 2.325289E+02 | global_load_balancing_loss: 1.000410E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11149.427 | params rms: 0.060371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:12:00.823504 | finish at 2026-10-17 19:32:23 + [2026-10-17 01:24:03] iteration 2093/ 2384 | consumed samples: 66976 | elapsed time per iteration (ms): 220739.7 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.95% | FLOPs estimator: workload-aware | logical tokens/s: 76004.5 | valid tokens/s: 76004.5 | physical tokens/s: 76857.8 | loss tokens/s: 57018.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.580691E-01 | lm-global-supervised-tokens: 1.258626E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.695273E-02 | lm-token-buffer-mib: 2.331150E+02 | global_load_balancing_loss: 1.000758E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11149.393 | params rms: 0.060370 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:50:35.244259 | finish at 2026-10-17 19:14:38 + [2026-10-17 01:27:45] iteration 2094/ 2384 | consumed samples: 67008 | elapsed time per iteration (ms): 221825.8 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.03% | FLOPs estimator: workload-aware | logical tokens/s: 75632.4 | valid tokens/s: 75632.4 | physical tokens/s: 76551.0 | loss tokens/s: 56842.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.035618E-01 | lm-global-supervised-tokens: 1.260917E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.436553E-02 | lm-token-buffer-mib: 2.368176E+02 | global_load_balancing_loss: 9.998862E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11149.359 | params rms: 0.060370 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:52:09.491041 | finish at 2026-10-17 19:19:54 + [2026-10-17 01:31:26] iteration 2095/ 2384 | consumed samples: 67040 | elapsed time per iteration (ms): 220920.1 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 75942.4 | valid tokens/s: 75942.4 | physical tokens/s: 76647.6 | loss tokens/s: 55653.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.904767E-01 | lm-global-supervised-tokens: 1.229488E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.647182E-02 | lm-token-buffer-mib: 2.330202E+02 | global_load_balancing_loss: 1.001041E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11149.324 | params rms: 0.060370 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:44:05.900004 | finish at 2026-10-17 19:15:32 + [2026-10-17 01:35:10] iteration 2096/ 2384 | consumed samples: 67072 | elapsed time per iteration (ms): 223995.6 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 74899.7 | valid tokens/s: 74899.7 | physical tokens/s: 75732.9 | loss tokens/s: 57363.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.993503E-01 | lm-global-supervised-tokens: 1.284921E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.755107E-02 | lm-token-buffer-mib: 2.342822E+02 | global_load_balancing_loss: 1.001043E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11149.288 | params rms: 0.060370 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:55:10.743027 | finish at 2026-10-17 19:30:20 + [2026-10-17 01:38:52] iteration 2097/ 2384 | consumed samples: 67104 | elapsed time per iteration (ms): 222153.6 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 75520.4 | valid tokens/s: 75520.4 | physical tokens/s: 76046.0 | loss tokens/s: 55296.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.707108E-01 | lm-global-supervised-tokens: 1.228426E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.251418E-02 | lm-token-buffer-mib: 2.350306E+02 | global_load_balancing_loss: 1.000056E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11149.256 | params rms: 0.060370 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:42:38.075872 | finish at 2026-10-17 19:21:30 + [2026-10-17 01:42:35] iteration 2098/ 2384 | consumed samples: 67136 | elapsed time per iteration (ms): 222946.3 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.24% | FLOPs estimator: workload-aware | logical tokens/s: 75252.2 | valid tokens/s: 75252.2 | physical tokens/s: 76009.9 | loss tokens/s: 56967.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.005484E-01 | lm-global-supervised-tokens: 1.270060E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.421147E-02 | lm-token-buffer-mib: 2.346653E+02 | global_load_balancing_loss: 1.000294E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11149.225 | params rms: 0.060369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:42:42.628921 | finish at 2026-10-17 19:25:17 + [2026-10-17 01:46:19] iteration 2099/ 2384 | consumed samples: 67168 | elapsed time per iteration (ms): 223957.3 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 74912.5 | valid tokens/s: 74912.5 | physical tokens/s: 76101.1 | loss tokens/s: 58413.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.837449E-01 | lm-global-supervised-tokens: 1.308216E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.971628E-02 | lm-token-buffer-mib: 2.372285E+02 | global_load_balancing_loss: 9.989343E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11149.195 | params rms: 0.060369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:43:47.825797 | finish at 2026-10-17 19:30:07 + [2026-10-17 01:50:25] iteration 2100/ 2384 | consumed samples: 67200 | elapsed time per iteration (ms): 246627.5 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 68026.3 | valid tokens/s: 68026.3 | physical tokens/s: 69013.4 | loss tokens/s: 53933.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.904126E-01 | lm-global-supervised-tokens: 1.330142E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.915448E-02 | lm-token-buffer-mib: 2.343763E+02 | global_load_balancing_loss: 1.000191E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11149.164 | params rms: 0.060369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:27:22.205818 | finish at 2026-10-17 21:17:48 + [2026-10-17 01:54:06] iteration 2101/ 2384 | consumed samples: 67232 | elapsed time per iteration (ms): 220655.4 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 76033.4 | valid tokens/s: 76033.4 | physical tokens/s: 76899.1 | loss tokens/s: 58547.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.840093E-01 | lm-global-supervised-tokens: 1.291883E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.264868E-02 | lm-token-buffer-mib: 2.336571E+02 | global_load_balancing_loss: 9.995576E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11149.135 | params rms: 0.060369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:20:45.464716 | finish at 2026-10-17 19:14:52 + [2026-10-17 01:57:55] iteration 2102/ 2384 | consumed samples: 67264 | elapsed time per iteration (ms): 228648.8 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 73375.4 | valid tokens/s: 73375.4 | physical tokens/s: 74489.5 | loss tokens/s: 53943.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.625893E-01 | lm-global-supervised-tokens: 1.233420E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.186025E-02 | lm-token-buffer-mib: 2.349768E+02 | global_load_balancing_loss: 1.000678E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11149.104 | params rms: 0.060369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:54:38.974480 | finish at 2026-10-17 19:52:34 + [2026-10-17 02:01:38] iteration 2103/ 2384 | consumed samples: 67296 | elapsed time per iteration (ms): 223335.2 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75121.1 | valid tokens/s: 75121.1 | physical tokens/s: 75538.5 | loss tokens/s: 56458.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.744891E-01 | lm-global-supervised-tokens: 1.260927E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.075890E-02 | lm-token-buffer-mib: 2.337472E+02 | global_load_balancing_loss: 9.995740E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11149.073 | params rms: 0.060369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:25:57.201202 | finish at 2026-10-17 19:27:35 + [2026-10-17 02:05:20] iteration 2104/ 2384 | consumed samples: 67328 | elapsed time per iteration (ms): 221707.8 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 75672.6 | valid tokens/s: 75672.5 | physical tokens/s: 76148.4 | loss tokens/s: 57632.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.362641E-01 | lm-global-supervised-tokens: 1.277756E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.510568E-02 | lm-token-buffer-mib: 2.341996E+02 | global_load_balancing_loss: 1.000794E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11149.042 | params rms: 0.060368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:14:38.178034 | finish at 2026-10-17 19:19:58 + [2026-10-17 02:09:02] iteration 2105/ 2384 | consumed samples: 67360 | elapsed time per iteration (ms): 222561.4 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 75382.3 | valid tokens/s: 75382.3 | physical tokens/s: 76466.2 | loss tokens/s: 56441.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.914627E-01 | lm-global-supervised-tokens: 1.256160E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.655443E-02 | lm-token-buffer-mib: 2.350759E+02 | global_load_balancing_loss: 1.000977E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11149.011 | params rms: 0.060368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:14:54.632511 | finish at 2026-10-17 19:23:57 + [2026-10-17 02:12:47] iteration 2106/ 2384 | consumed samples: 67392 | elapsed time per iteration (ms): 224614.7 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 74693.3 | valid tokens/s: 74693.3 | physical tokens/s: 75227.0 | loss tokens/s: 58805.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.776562E-01 | lm-global-supervised-tokens: 1.320866E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.632849E-02 | lm-token-buffer-mib: 2.332365E+02 | global_load_balancing_loss: 1.001506E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11148.980 | params rms: 0.060368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:20:42.889074 | finish at 2026-10-17 19:33:30 + [2026-10-17 02:16:29] iteration 2107/ 2384 | consumed samples: 67424 | elapsed time per iteration (ms): 222217.7 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 75498.8 | valid tokens/s: 75498.8 | physical tokens/s: 76199.1 | loss tokens/s: 57310.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.959680E-01 | lm-global-supervised-tokens: 1.273546E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.283659E-02 | lm-token-buffer-mib: 2.332350E+02 | global_load_balancing_loss: 1.001347E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11148.950 | params rms: 0.060368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:05:54.303496 | finish at 2026-10-17 19:22:23 + [2026-10-17 02:20:13] iteration 2108/ 2384 | consumed samples: 67456 | elapsed time per iteration (ms): 223684.6 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 75003.8 | valid tokens/s: 75003.8 | physical tokens/s: 75506.9 | loss tokens/s: 55891.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.150554E-01 | lm-global-supervised-tokens: 1.250204E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.590570E-02 | lm-token-buffer-mib: 2.346701E+02 | global_load_balancing_loss: 9.999435E-01 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11148.921 | params rms: 0.060368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:08:56.948381 | finish at 2026-10-17 19:29:10 + [2026-10-17 02:23:57] iteration 2109/ 2384 | consumed samples: 67488 | elapsed time per iteration (ms): 223652.5 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.28% | FLOPs estimator: workload-aware | logical tokens/s: 75014.7 | valid tokens/s: 75014.7 | physical tokens/s: 75437.6 | loss tokens/s: 57958.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.288077E-01 | lm-global-supervised-tokens: 1.296253E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.195869E-02 | lm-token-buffer-mib: 2.334222E+02 | global_load_balancing_loss: 1.000178E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11148.889 | params rms: 0.060368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:05:04.441082 | finish at 2026-10-17 19:29:01 + [2026-10-17 02:28:02] iteration 2110/ 2384 | consumed samples: 67520 | elapsed time per iteration (ms): 245639.4 | throughput per GPU (TFLOP/s/GPU): 17.8 | MFU 5.72% | FLOPs estimator: workload-aware | logical tokens/s: 68300.1 | valid tokens/s: 68300.1 | physical tokens/s: 69198.0 | loss tokens/s: 54319.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.440075E-01 | lm-global-supervised-tokens: 1.334310E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.490736E-02 | lm-token-buffer-mib: 2.340833E+02 | global_load_balancing_loss: 9.997463E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11148.852 | params rms: 0.060367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:41:45.207818 | finish at 2026-10-17 21:09:47 + [2026-10-17 02:31:47] iteration 2111/ 2384 | consumed samples: 67552 | elapsed time per iteration (ms): 225046.6 | throughput per GPU (TFLOP/s/GPU): 25.4 | MFU 8.15% | FLOPs estimator: workload-aware | logical tokens/s: 74549.8 | valid tokens/s: 74549.8 | physical tokens/s: 74857.4 | loss tokens/s: 54308.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.758167E-01 | lm-global-supervised-tokens: 1.222186E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.624123E-02 | lm-token-buffer-mib: 2.329364E+02 | global_load_balancing_loss: 9.993583E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11148.817 | params rms: 0.060367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:03:57.718899 | finish at 2026-10-17 19:35:45 + [2026-10-17 02:35:30] iteration 2112/ 2384 | consumed samples: 67584 | elapsed time per iteration (ms): 222521.6 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.80% | FLOPs estimator: workload-aware | logical tokens/s: 75395.8 | valid tokens/s: 75395.8 | physical tokens/s: 76183.4 | loss tokens/s: 60537.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.518953E-01 | lm-global-supervised-tokens: 1.347079E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.390938E-02 | lm-token-buffer-mib: 2.329159E+02 | global_load_balancing_loss: 9.989558E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11148.786 | params rms: 0.060367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:48:45.873581 | finish at 2026-10-17 19:24:16 + [2026-10-17 02:39:15] iteration 2113/ 2384 | consumed samples: 67616 | elapsed time per iteration (ms): 225286.7 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 74470.4 | valid tokens/s: 74470.4 | physical tokens/s: 74992.7 | loss tokens/s: 56644.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.404223E-01 | lm-global-supervised-tokens: 1.276127E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.640540E-02 | lm-token-buffer-mib: 2.331731E+02 | global_load_balancing_loss: 1.001130E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11148.760 | params rms: 0.060367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:57:32.686076 | finish at 2026-10-17 19:36:48 + [2026-10-17 02:42:55] iteration 2114/ 2384 | consumed samples: 67648 | elapsed time per iteration (ms): 219656.1 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 76379.4 | valid tokens/s: 76379.4 | physical tokens/s: 77889.9 | loss tokens/s: 57327.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.657475E-01 | lm-global-supervised-tokens: 1.259225E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.827881E-02 | lm-token-buffer-mib: 2.340884E+02 | global_load_balancing_loss: 1.000376E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11148.738 | params rms: 0.060367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:28:27.151580 | finish at 2026-10-17 19:11:22 + [2026-10-17 02:46:38] iteration 2115/ 2384 | consumed samples: 67680 | elapsed time per iteration (ms): 222923.4 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 75260.0 | valid tokens/s: 75260.0 | physical tokens/s: 76146.8 | loss tokens/s: 59589.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.606621E-01 | lm-global-supervised-tokens: 1.328386E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.150596E-02 | lm-token-buffer-mib: 2.351248E+02 | global_load_balancing_loss: 9.996673E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11148.716 | params rms: 0.060367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:39:26.387333 | finish at 2026-10-17 19:26:04 + [2026-10-17 02:50:20] iteration 2116/ 2384 | consumed samples: 67712 | elapsed time per iteration (ms): 221938.2 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 75593.9 | valid tokens/s: 75593.9 | physical tokens/s: 76366.6 | loss tokens/s: 59715.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.229554E-01 | lm-global-supervised-tokens: 1.325317E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.280313E-02 | lm-token-buffer-mib: 2.346736E+02 | global_load_balancing_loss: 1.000936E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11148.691 | params rms: 0.060367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:31:19.427823 | finish at 2026-10-17 19:21:39 + [2026-10-17 02:54:01] iteration 2117/ 2384 | consumed samples: 67744 | elapsed time per iteration (ms): 221422.8 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 75770.0 | valid tokens/s: 75770.0 | physical tokens/s: 76434.6 | loss tokens/s: 59452.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.108285E-01 | lm-global-supervised-tokens: 1.316412E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.387864E-02 | lm-token-buffer-mib: 2.339328E+02 | global_load_balancing_loss: 1.001748E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11148.668 | params rms: 0.060366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:25:19.886790 | finish at 2026-10-17 19:19:21 + [2026-10-17 02:57:42] iteration 2118/ 2384 | consumed samples: 67776 | elapsed time per iteration (ms): 221420.3 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 75770.8 | valid tokens/s: 75770.8 | physical tokens/s: 77042.1 | loss tokens/s: 58243.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.969320E-01 | lm-global-supervised-tokens: 1.289621E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.346905E-02 | lm-token-buffer-mib: 2.351046E+02 | global_load_balancing_loss: 1.001365E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11148.647 | params rms: 0.060366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:21:37.811408 | finish at 2026-10-17 19:19:20 + [2026-10-17 03:01:26] iteration 2119/ 2384 | consumed samples: 67808 | elapsed time per iteration (ms): 224039.5 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 74885.0 | valid tokens/s: 74885.0 | physical tokens/s: 76278.3 | loss tokens/s: 55342.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.415414E-01 | lm-global-supervised-tokens: 1.239884E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.719884E-02 | lm-token-buffer-mib: 2.344816E+02 | global_load_balancing_loss: 1.001046E+00 | loss scale: 1.0 | grad norm: 0.238 | params norm: 11148.628 | params rms: 0.060366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:29:30.476029 | finish at 2026-10-17 19:30:57 + [2026-10-17 03:05:34] iteration 2120/ 2384 | consumed samples: 67840 | elapsed time per iteration (ms): 248067.6 | throughput per GPU (TFLOP/s/GPU): 18.4 | MFU 5.90% | FLOPs estimator: workload-aware | logical tokens/s: 67631.6 | valid tokens/s: 67631.6 | physical tokens/s: 68452.1 | loss tokens/s: 51489.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.320377E-01 | lm-global-supervised-tokens: 1.277296E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.203087E-02 | lm-token-buffer-mib: 2.367964E+02 | global_load_balancing_loss: 1.000798E+00 | loss scale: 1.0 | grad norm: 0.342 | params norm: 11148.607 | params rms: 0.060366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:11:29.834255 | finish at 2026-10-17 21:17:04 +times across ranks (ms): + save-checkpoint: + rank 0: 24330.55 + rank 1: 24330.57 + rank 2: 24330.58 + rank 3: 24330.62 + rank 4: 24330.57 + rank 5: 24330.59 + rank 6: 24330.56 + rank 7: 24330.56 + rank 8: 24331.49 + rank 9: 24331.53 + rank 10: 24331.57 + rank 11: 24331.49 + rank 12: 24331.52 + rank 13: 24331.49 + rank 14: 24331.51 + rank 15: 24331.48 + rank 16: 24330.45 + rank 17: 24330.47 + rank 18: 24330.45 + rank 19: 24330.45 + rank 20: 24330.45 + rank 21: 24330.47 + rank 22: 24330.49 + rank 23: 24330.51 + rank 24: 24330.42 + rank 25: 24330.43 + rank 26: 24330.42 + rank 27: 24330.44 + rank 28: 24330.40 + rank 29: 24330.47 + rank 30: 24330.43 + rank 31: 24330.42 + rank 32: 24330.37 + rank 33: 24330.38 + rank 34: 24330.36 + rank 35: 24330.39 + rank 36: 24330.33 + rank 37: 24330.36 + rank 38: 24330.32 + rank 39: 24330.34 + rank 40: 24331.74 + rank 41: 24331.74 + rank 42: 24331.73 + rank 43: 24331.79 + rank 44: 24331.74 + rank 45: 24331.74 + rank 46: 24331.73 + rank 47: 24331.73 + rank 48: 24330.69 + rank 49: 24330.71 + rank 50: 24330.71 + rank 51: 24331.12 + rank 52: 24330.67 + rank 53: 24330.71 + rank 54: 24330.69 + rank 55: 24330.68 + rank 56: 24330.57 + rank 57: 24330.56 + rank 58: 24330.58 + rank 59: 24330.55 + rank 60: 24330.60 + rank 61: 24330.55 + rank 62: 24330.55 + rank 63: 24330.57 + [2026-10-17 03:09:35] iteration 2121/ 2384 | consumed samples: 67872 | elapsed time per iteration (ms): 216572.7 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 77466.7 | valid tokens/s: 77466.7 | physical tokens/s: 78061.2 | loss tokens/s: 57576.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.749724E-01 | lm-global-supervised-tokens: 1.246952E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.380630E-02 | lm-token-buffer-mib: 2.327495E+02 | global_load_balancing_loss: 9.998278E-01 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11148.586 | params rms: 0.060366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:49:18.628195 | finish at 2026-10-17 18:58:54 + [2026-10-17 03:13:25] iteration 2122/ 2384 | consumed samples: 67904 | elapsed time per iteration (ms): 229326.2 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.94% | FLOPs estimator: workload-aware | logical tokens/s: 73158.4 | valid tokens/s: 73158.4 | physical tokens/s: 74076.3 | loss tokens/s: 57014.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.936467E-01 | lm-global-supervised-tokens: 1.307491E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.855624E-02 | lm-token-buffer-mib: 2.358594E+02 | global_load_balancing_loss: 9.976819E-01 | loss scale: 1.0 | grad norm: 0.663 | params norm: 11148.558 | params rms: 0.060366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:41:23.474771 | finish at 2026-10-17 19:54:48 + [2026-10-17 03:17:07] iteration 2123/ 2384 | consumed samples: 67936 | elapsed time per iteration (ms): 221842.5 | throughput per GPU (TFLOP/s/GPU): 26.3 | MFU 8.43% | FLOPs estimator: workload-aware | logical tokens/s: 75626.5 | valid tokens/s: 75626.5 | physical tokens/s: 76041.6 | loss tokens/s: 55048.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.048931E-01 | lm-global-supervised-tokens: 1.221198E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.611360E-02 | lm-token-buffer-mib: 2.328420E+02 | global_load_balancing_loss: 9.995933E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11148.526 | params rms: 0.060366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:05:00.881354 | finish at 2026-10-17 19:22:08 + [2026-10-17 03:20:49] iteration 2124/ 2384 | consumed samples: 67968 | elapsed time per iteration (ms): 221977.4 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.68% | FLOPs estimator: workload-aware | logical tokens/s: 75580.7 | valid tokens/s: 75580.7 | physical tokens/s: 76106.1 | loss tokens/s: 57935.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.327871E-01 | lm-global-supervised-tokens: 1.286029E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.086130E-02 | lm-token-buffer-mib: 2.325634E+02 | global_load_balancing_loss: 9.992623E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11148.494 | params rms: 0.060365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:01:54.119058 | finish at 2026-10-17 19:22:43 + [2026-10-17 03:24:29] iteration 2125/ 2384 | consumed samples: 68000 | elapsed time per iteration (ms): 220232.7 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 76179.5 | valid tokens/s: 76179.5 | physical tokens/s: 76643.9 | loss tokens/s: 55532.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.483018E-01 | lm-global-supervised-tokens: 1.223005E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.266322E-02 | lm-token-buffer-mib: 2.334194E+02 | global_load_balancing_loss: 1.001405E+00 | loss scale: 1.0 | grad norm: 0.248 | params norm: 11148.467 | params rms: 0.060365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:50:40.269143 | finish at 2026-10-17 19:15:09 + [2026-10-17 03:28:11] iteration 2126/ 2384 | consumed samples: 68032 | elapsed time per iteration (ms): 222501.9 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 75402.5 | valid tokens/s: 75402.5 | physical tokens/s: 75911.7 | loss tokens/s: 56167.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.225607E-01 | lm-global-supervised-tokens: 1.249740E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.225131E-02 | lm-token-buffer-mib: 2.330367E+02 | global_load_balancing_loss: 1.000872E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11148.439 | params rms: 0.060365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:56:45.490189 | finish at 2026-10-17 19:24:57 + [2026-10-17 03:31:53] iteration 2127/ 2384 | consumed samples: 68064 | elapsed time per iteration (ms): 221585.6 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 75714.3 | valid tokens/s: 75714.3 | physical tokens/s: 76471.7 | loss tokens/s: 59488.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.068140E-01 | lm-global-supervised-tokens: 1.318177E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.874967E-02 | lm-token-buffer-mib: 2.342485E+02 | global_load_balancing_loss: 9.999442E-01 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11148.412 | params rms: 0.060365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:49:07.503402 | finish at 2026-10-17 19:21:00 + [2026-10-17 03:35:33] iteration 2128/ 2384 | consumed samples: 68096 | elapsed time per iteration (ms): 219841.3 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 76315.1 | valid tokens/s: 76315.1 | physical tokens/s: 77376.8 | loss tokens/s: 57789.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.521105E-01 | lm-global-supervised-tokens: 1.270444E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.279986E-02 | lm-token-buffer-mib: 2.332306E+02 | global_load_balancing_loss: 1.001276E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11148.388 | params rms: 0.060365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:37:59.376038 | finish at 2026-10-17 19:13:32 + [2026-10-17 03:39:14] iteration 2129/ 2384 | consumed samples: 68128 | elapsed time per iteration (ms): 221359.8 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 75791.4 | valid tokens/s: 75791.4 | physical tokens/s: 76862.8 | loss tokens/s: 57598.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.758108E-01 | lm-global-supervised-tokens: 1.275002E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.392590E-02 | lm-token-buffer-mib: 2.337538E+02 | global_load_balancing_loss: 1.000778E+00 | loss scale: 1.0 | grad norm: 0.204 | params norm: 11148.365 | params rms: 0.060365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:40:46.760091 | finish at 2026-10-17 19:20:01 + [2026-10-17 03:43:17] iteration 2130/ 2384 | consumed samples: 68160 | elapsed time per iteration (ms): 243170.7 | throughput per GPU (TFLOP/s/GPU): 17.7 | MFU 5.68% | FLOPs estimator: workload-aware | logical tokens/s: 68993.5 | valid tokens/s: 68993.5 | physical tokens/s: 69363.1 | loss tokens/s: 52088.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.807925E-01 | lm-global-supervised-tokens: 1.266628E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.761490E-02 | lm-token-buffer-mib: 2.339006E+02 | global_load_balancing_loss: 1.002087E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11148.345 | params rms: 0.060365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:09:25.347766 | finish at 2026-10-17 20:52:43 + [2026-10-17 03:47:00] iteration 2131/ 2384 | consumed samples: 68192 | elapsed time per iteration (ms): 222774.0 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.32% | FLOPs estimator: workload-aware | logical tokens/s: 75310.3 | valid tokens/s: 75310.3 | physical tokens/s: 75767.2 | loss tokens/s: 56693.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.459333E-01 | lm-global-supervised-tokens: 1.262973E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.249007E-02 | lm-token-buffer-mib: 2.337645E+02 | global_load_balancing_loss: 9.992173E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11148.325 | params rms: 0.060365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:39:21.831090 | finish at 2026-10-17 19:26:22 + [2026-10-17 03:50:44] iteration 2132/ 2384 | consumed samples: 68224 | elapsed time per iteration (ms): 223451.3 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 75082.0 | valid tokens/s: 75082.0 | physical tokens/s: 76236.2 | loss tokens/s: 60873.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.915222E-01 | lm-global-supervised-tokens: 1.360224E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.415193E-02 | lm-token-buffer-mib: 2.356952E+02 | global_load_balancing_loss: 9.985188E-01 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11148.303 | params rms: 0.060364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:38:29.720306 | finish at 2026-10-17 19:29:13 + [2026-10-17 03:54:31] iteration 2133/ 2384 | consumed samples: 68256 | elapsed time per iteration (ms): 227201.7 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 73842.8 | valid tokens/s: 73842.8 | physical tokens/s: 74636.1 | loss tokens/s: 57424.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.528410E-01 | lm-global-supervised-tokens: 1.304692E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.003322E-02 | lm-token-buffer-mib: 2.343966E+02 | global_load_balancing_loss: 9.999650E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11148.280 | params rms: 0.060364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:50:27.626035 | finish at 2026-10-17 19:44:58 + [2026-10-17 03:58:11] iteration 2134/ 2384 | consumed samples: 68288 | elapsed time per iteration (ms): 220240.6 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.49% | FLOPs estimator: workload-aware | logical tokens/s: 76176.6 | valid tokens/s: 76176.6 | physical tokens/s: 76803.3 | loss tokens/s: 58637.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.264994E-01 | lm-global-supervised-tokens: 1.291434E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.122977E-02 | lm-token-buffer-mib: 2.337065E+02 | global_load_balancing_loss: 9.990881E-01 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11148.258 | params rms: 0.060364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:17:40.156465 | finish at 2026-10-17 19:15:51 + [2026-10-17 04:01:57] iteration 2135/ 2384 | consumed samples: 68320 | elapsed time per iteration (ms): 226169.1 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 74179.7 | valid tokens/s: 74179.7 | physical tokens/s: 75739.4 | loss tokens/s: 53840.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.603417E-01 | lm-global-supervised-tokens: 1.217706E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.145657E-02 | lm-token-buffer-mib: 2.361938E+02 | global_load_balancing_loss: 1.000978E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11148.234 | params rms: 0.060364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:38:36.116538 | finish at 2026-10-17 19:40:33 + [2026-10-17 04:05:43] iteration 2136/ 2384 | consumed samples: 68352 | elapsed time per iteration (ms): 225612.3 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 74362.9 | valid tokens/s: 74362.9 | physical tokens/s: 75031.6 | loss tokens/s: 54038.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.673044E-01 | lm-global-supervised-tokens: 1.219164E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.332909E-02 | lm-token-buffer-mib: 2.344656E+02 | global_load_balancing_loss: 1.001583E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11148.210 | params rms: 0.060364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:32:31.854401 | finish at 2026-10-17 19:38:15 + [2026-10-17 04:09:23] iteration 2137/ 2384 | consumed samples: 68384 | elapsed time per iteration (ms): 220573.3 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 76061.7 | valid tokens/s: 76061.7 | physical tokens/s: 76672.0 | loss tokens/s: 57777.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.285895E-01 | lm-global-supervised-tokens: 1.274422E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.428471E-02 | lm-token-buffer-mib: 2.331077E+02 | global_load_balancing_loss: 1.000998E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11148.183 | params rms: 0.060364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:08:01.615613 | finish at 2026-10-17 19:17:25 + [2026-10-17 04:13:08] iteration 2138/ 2384 | consumed samples: 68416 | elapsed time per iteration (ms): 224365.4 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 74776.2 | valid tokens/s: 74776.2 | physical tokens/s: 75882.8 | loss tokens/s: 59149.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.778794E-01 | lm-global-supervised-tokens: 1.327100E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.237923E-02 | lm-token-buffer-mib: 2.363474E+02 | global_load_balancing_loss: 1.000607E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11148.157 | params rms: 0.060364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:19:53.888125 | finish at 2026-10-17 19:33:02 + [2026-10-17 04:16:48] iteration 2139/ 2384 | consumed samples: 68448 | elapsed time per iteration (ms): 220546.4 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 76071.0 | valid tokens/s: 76071.0 | physical tokens/s: 76662.5 | loss tokens/s: 54740.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.377442E-01 | lm-global-supervised-tokens: 1.207281E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.273732E-02 | lm-token-buffer-mib: 2.332941E+02 | global_load_balancing_loss: 9.997954E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11148.129 | params rms: 0.060363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:00:33.870938 | finish at 2026-10-17 19:17:22 + [2026-10-17 04:20:54] iteration 2140/ 2384 | consumed samples: 68480 | elapsed time per iteration (ms): 245331.4 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 68385.9 | valid tokens/s: 68385.9 | physical tokens/s: 68856.5 | loss tokens/s: 50956.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.537523E-01 | lm-global-supervised-tokens: 1.250112E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.579161E-02 | lm-token-buffer-mib: 2.339543E+02 | global_load_balancing_loss: 1.000050E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11148.099 | params rms: 0.060363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:37:40.867921 | finish at 2026-10-17 20:58:34 + [2026-10-17 04:24:34] iteration 2141/ 2384 | consumed samples: 68512 | elapsed time per iteration (ms): 220299.2 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 76156.5 | valid tokens/s: 76156.5 | physical tokens/s: 76421.2 | loss tokens/s: 55997.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.394722E-01 | lm-global-supervised-tokens: 1.233612E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.704532E-02 | lm-token-buffer-mib: 2.324727E+02 | global_load_balancing_loss: 1.000340E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11148.066 | params rms: 0.060363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:52:12.700226 | finish at 2026-10-17 19:16:47 + [2026-10-17 04:28:15] iteration 2142/ 2384 | consumed samples: 68544 | elapsed time per iteration (ms): 221494.5 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75745.4 | valid tokens/s: 75745.4 | physical tokens/s: 76380.0 | loss tokens/s: 58341.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.291944E-01 | lm-global-supervised-tokens: 1.292227E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.478996E-02 | lm-token-buffer-mib: 2.332781E+02 | global_load_balancing_loss: 1.000622E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11148.034 | params rms: 0.060363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:53:21.668808 | finish at 2026-10-17 19:21:37 + [2026-10-17 04:32:00] iteration 2143/ 2384 | consumed samples: 68576 | elapsed time per iteration (ms): 224426.3 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.53% | FLOPs estimator: workload-aware | logical tokens/s: 74755.9 | valid tokens/s: 74755.9 | physical tokens/s: 75569.3 | loss tokens/s: 57410.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.208773E-01 | lm-global-supervised-tokens: 1.288442E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.676859E-02 | lm-token-buffer-mib: 2.353710E+02 | global_load_balancing_loss: 1.001896E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11148.003 | params rms: 0.060363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:01:26.747218 | finish at 2026-10-17 19:33:27 + [2026-10-17 04:35:46] iteration 2144/ 2384 | consumed samples: 68608 | elapsed time per iteration (ms): 226589.7 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.28% | FLOPs estimator: workload-aware | logical tokens/s: 74042.2 | valid tokens/s: 74042.2 | physical tokens/s: 75372.6 | loss tokens/s: 59050.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.498068E-01 | lm-global-supervised-tokens: 1.338028E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.521856E-02 | lm-token-buffer-mib: 2.354907E+02 | global_load_balancing_loss: 9.999157E-01 | loss scale: 1.0 | grad norm: 0.454 | params norm: 11147.972 | params rms: 0.060363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:06:21.527824 | finish at 2026-10-17 19:42:08 + [2026-10-17 04:39:31] iteration 2145/ 2384 | consumed samples: 68640 | elapsed time per iteration (ms): 224836.0 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 74619.7 | valid tokens/s: 74619.7 | physical tokens/s: 75032.3 | loss tokens/s: 52478.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.423627E-01 | lm-global-supervised-tokens: 1.179905E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.242889E-02 | lm-token-buffer-mib: 2.334920E+02 | global_load_balancing_loss: 1.000239E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11147.940 | params rms: 0.060362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:55:35.797895 | finish at 2026-10-17 19:35:07 + [2026-10-17 04:43:15] iteration 2146/ 2384 | consumed samples: 68672 | elapsed time per iteration (ms): 223609.3 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 75029.0 | valid tokens/s: 75029.0 | physical tokens/s: 75770.5 | loss tokens/s: 58526.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.723985E-01 | lm-global-supervised-tokens: 1.308702E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.788048E-02 | lm-token-buffer-mib: 2.351387E+02 | global_load_balancing_loss: 1.000571E+00 | loss scale: 1.0 | grad norm: 0.402 | params norm: 11147.905 | params rms: 0.060362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:46:59.007304 | finish at 2026-10-17 19:30:14 + [2026-10-17 04:46:58] iteration 2147/ 2384 | consumed samples: 68704 | elapsed time per iteration (ms): 223545.6 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 75050.5 | valid tokens/s: 75050.5 | physical tokens/s: 75766.0 | loss tokens/s: 57085.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.218658E-01 | lm-global-supervised-tokens: 1.276115E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.187015E-02 | lm-token-buffer-mib: 2.352521E+02 | global_load_balancing_loss: 9.985206E-01 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11147.872 | params rms: 0.060362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:43:00.301478 | finish at 2026-10-17 19:29:59 + [2026-10-17 04:50:40] iteration 2148/ 2384 | consumed samples: 68736 | elapsed time per iteration (ms): 221817.4 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.47% | FLOPs estimator: workload-aware | logical tokens/s: 75635.1 | valid tokens/s: 75635.1 | physical tokens/s: 76493.9 | loss tokens/s: 60624.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.413258E-01 | lm-global-supervised-tokens: 1.344750E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.821183E-02 | lm-token-buffer-mib: 2.340170E+02 | global_load_balancing_loss: 9.997127E-01 | loss scale: 1.0 | grad norm: 0.266 | params norm: 11147.841 | params rms: 0.060362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:32:28.911684 | finish at 2026-10-17 19:23:09 + [2026-10-17 04:54:24] iteration 2149/ 2384 | consumed samples: 68768 | elapsed time per iteration (ms): 223290.4 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 75136.2 | valid tokens/s: 75136.2 | physical tokens/s: 75897.0 | loss tokens/s: 55925.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.451675E-01 | lm-global-supervised-tokens: 1.248764E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.009572E-02 | lm-token-buffer-mib: 2.354740E+02 | global_load_balancing_loss: 1.000792E+00 | loss scale: 1.0 | grad norm: 0.257 | params norm: 11147.813 | params rms: 0.060362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:34:33.251514 | finish at 2026-10-17 19:28:57 + [2026-10-17 04:58:29] iteration 2150/ 2384 | consumed samples: 68800 | elapsed time per iteration (ms): 245860.4 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.14% | FLOPs estimator: workload-aware | logical tokens/s: 68238.7 | valid tokens/s: 68238.7 | physical tokens/s: 69236.3 | loss tokens/s: 50146.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.266447E-01 | lm-global-supervised-tokens: 1.232906E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.883623E-02 | lm-token-buffer-mib: 2.354505E+02 | global_load_balancing_loss: 1.000806E+00 | loss scale: 1.0 | grad norm: 0.203 | params norm: 11147.784 | params rms: 0.060362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:58:51.326506 | finish at 2026-10-17 20:57:21 + [2026-10-17 05:02:11] iteration 2151/ 2384 | consumed samples: 68832 | elapsed time per iteration (ms): 221328.8 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 75802.0 | valid tokens/s: 75802.0 | physical tokens/s: 76670.9 | loss tokens/s: 57856.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.162387E-01 | lm-global-supervised-tokens: 1.280526E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.684859E-02 | lm-token-buffer-mib: 2.328028E+02 | global_load_balancing_loss: 1.000153E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11147.749 | params rms: 0.060361 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:19:29.609503 | finish at 2026-10-17 19:21:40 + [2026-10-17 05:05:54] iteration 2152/ 2384 | consumed samples: 68864 | elapsed time per iteration (ms): 222990.4 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 75237.3 | valid tokens/s: 75237.3 | physical tokens/s: 75608.5 | loss tokens/s: 58883.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.291235E-01 | lm-global-supervised-tokens: 1.313035E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.416144E-02 | lm-token-buffer-mib: 2.325491E+02 | global_load_balancing_loss: 1.000439E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11147.713 | params rms: 0.060361 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:22:13.767508 | finish at 2026-10-17 19:28:07 + [2026-10-17 05:09:40] iteration 2153/ 2384 | consumed samples: 68896 | elapsed time per iteration (ms): 225975.6 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.30% | FLOPs estimator: workload-aware | logical tokens/s: 74243.4 | valid tokens/s: 74243.4 | physical tokens/s: 74986.1 | loss tokens/s: 55749.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.574647E-01 | lm-global-supervised-tokens: 1.259791E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.228710E-02 | lm-token-buffer-mib: 2.349003E+02 | global_load_balancing_loss: 9.990520E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11147.678 | params rms: 0.060361 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:30:00.371091 | finish at 2026-10-17 19:39:40 + [2026-10-17 05:13:21] iteration 2154/ 2384 | consumed samples: 68928 | elapsed time per iteration (ms): 221369.6 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 75788.2 | valid tokens/s: 75788.2 | physical tokens/s: 76612.2 | loss tokens/s: 56790.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.051144E-01 | lm-global-supervised-tokens: 1.257168E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.751619E-02 | lm-token-buffer-mib: 2.354345E+02 | global_load_balancing_loss: 9.996211E-01 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11147.642 | params rms: 0.060361 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:08:35.002365 | finish at 2026-10-17 19:21:56 + [2026-10-17 05:17:03] iteration 2155/ 2384 | consumed samples: 68960 | elapsed time per iteration (ms): 221478.1 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 75751.1 | valid tokens/s: 75751.1 | physical tokens/s: 76510.0 | loss tokens/s: 55162.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.846218E-01 | lm-global-supervised-tokens: 1.221738E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.650252E-02 | lm-token-buffer-mib: 2.326612E+02 | global_load_balancing_loss: 1.000385E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11147.605 | params rms: 0.060361 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:05:18.480219 | finish at 2026-10-17 19:22:21 + [2026-10-17 05:20:42] iteration 2156/ 2384 | consumed samples: 68992 | elapsed time per iteration (ms): 219860.2 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 76308.5 | valid tokens/s: 76308.5 | physical tokens/s: 76823.5 | loss tokens/s: 57325.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.526750E-01 | lm-global-supervised-tokens: 1.260353E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.357187E-02 | lm-token-buffer-mib: 2.330131E+02 | global_load_balancing_loss: 1.002036E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11147.569 | params rms: 0.060360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:55:28.117973 | finish at 2026-10-17 19:16:10 + [2026-10-17 05:24:28] iteration 2157/ 2384 | consumed samples: 69024 | elapsed time per iteration (ms): 225330.7 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.88% | FLOPs estimator: workload-aware | logical tokens/s: 74455.9 | valid tokens/s: 74455.9 | physical tokens/s: 75199.8 | loss tokens/s: 52513.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.085759E-01 | lm-global-supervised-tokens: 1.183287E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.820103E-02 | lm-token-buffer-mib: 2.360838E+02 | global_load_balancing_loss: 1.002743E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11147.536 | params rms: 0.060360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:12:30.065959 | finish at 2026-10-17 19:36:58 + [2026-10-17 05:28:09] iteration 2158/ 2384 | consumed samples: 69056 | elapsed time per iteration (ms): 221311.3 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.66% | FLOPs estimator: workload-aware | logical tokens/s: 75808.0 | valid tokens/s: 75808.0 | physical tokens/s: 76599.2 | loss tokens/s: 58918.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.774217E-01 | lm-global-supervised-tokens: 1.303935E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.437097E-02 | lm-token-buffer-mib: 2.331714E+02 | global_load_balancing_loss: 1.001462E+00 | loss scale: 1.0 | grad norm: 0.323 | params norm: 11147.505 | params rms: 0.060360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:53:36.345996 | finish at 2026-10-17 19:21:45 + [2026-10-17 05:31:53] iteration 2159/ 2384 | consumed samples: 69088 | elapsed time per iteration (ms): 224039.9 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.28% | FLOPs estimator: workload-aware | logical tokens/s: 74884.8 | valid tokens/s: 74884.8 | physical tokens/s: 75371.1 | loss tokens/s: 53043.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.557179E-01 | lm-global-supervised-tokens: 1.188389E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.642691E-02 | lm-token-buffer-mib: 2.334741E+02 | global_load_balancing_loss: 1.000400E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11147.474 | params rms: 0.060360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:00:08.975905 | finish at 2026-10-17 19:32:02 + [2026-10-17 05:36:02] iteration 2160/ 2384 | consumed samples: 69120 | elapsed time per iteration (ms): 248554.1 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 67499.1 | valid tokens/s: 67499.1 | physical tokens/s: 67736.7 | loss tokens/s: 48372.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.129540E-01 | lm-global-supervised-tokens: 1.202306E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.629336E-02 | lm-token-buffer-mib: 2.330240E+02 | global_load_balancing_loss: 9.980429E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11147.442 | params rms: 0.060360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:27:56.122681 | finish at 2026-10-17 21:03:58 +times across ranks (ms): + save-checkpoint: + rank 0: 44933.11 + rank 1: 44933.12 + rank 2: 44933.07 + rank 3: 44933.07 + rank 4: 44933.09 + rank 5: 44932.99 + rank 6: 44933.11 + rank 7: 44933.12 + rank 8: 44591.45 + rank 9: 44591.39 + rank 10: 44591.44 + rank 11: 44591.41 + rank 12: 44591.41 + rank 13: 44591.41 + rank 14: 44591.37 + rank 15: 44591.44 + rank 16: 44591.36 + rank 17: 44591.38 + rank 18: 44591.36 + rank 19: 44591.41 + rank 20: 44591.39 + rank 21: 44591.39 + rank 22: 44591.39 + rank 23: 44591.35 + rank 24: 44590.68 + rank 25: 44590.68 + rank 26: 44590.68 + rank 27: 44590.54 + rank 28: 44590.60 + rank 29: 44590.67 + rank 30: 44590.65 + rank 31: 44590.61 + rank 32: 44590.57 + rank 33: 44590.49 + rank 34: 44590.47 + rank 35: 44590.48 + rank 36: 44590.44 + rank 37: 44590.48 + rank 38: 44590.42 + rank 39: 44590.43 + rank 40: 44591.60 + rank 41: 44591.63 + rank 42: 44591.60 + rank 43: 44591.59 + rank 44: 44591.59 + rank 45: 44591.61 + rank 46: 44591.60 + rank 47: 44591.61 + rank 48: 44590.96 + rank 49: 44591.12 + rank 50: 44591.09 + rank 51: 44591.08 + rank 52: 44590.96 + rank 53: 44591.01 + rank 54: 44591.05 + rank 55: 44590.98 + rank 56: 44590.46 + rank 57: 44590.52 + rank 58: 44590.54 + rank 59: 44590.51 + rank 60: 44590.52 + rank 61: 44590.47 + rank 62: 44590.46 + rank 63: 44590.46 + [2026-10-17 05:40:34] iteration 2161/ 2384 | consumed samples: 69152 | elapsed time per iteration (ms): 222398.9 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 75437.3 | valid tokens/s: 75437.3 | physical tokens/s: 75974.2 | loss tokens/s: 59919.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.573752E-01 | lm-global-supervised-tokens: 1.332598E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.456062E-01 | lm-token-buffer-mib: 2.332933E+02 | global_load_balancing_loss: 1.000070E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11147.407 | params rms: 0.060360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:46:34.950241 | finish at 2026-10-17 19:27:09 + [2026-10-17 05:44:19] iteration 2162/ 2384 | consumed samples: 69184 | elapsed time per iteration (ms): 224660.7 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 74677.9 | valid tokens/s: 74677.9 | physical tokens/s: 75378.5 | loss tokens/s: 54941.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.353574E-01 | lm-global-supervised-tokens: 1.234317E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.380177E-02 | lm-token-buffer-mib: 2.343587E+02 | global_load_balancing_loss: 9.993462E-01 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11147.373 | params rms: 0.060359 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:51:14.682564 | finish at 2026-10-17 19:35:33 + [2026-10-17 05:48:00] iteration 2163/ 2384 | consumed samples: 69216 | elapsed time per iteration (ms): 221351.5 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.25% | FLOPs estimator: workload-aware | logical tokens/s: 75794.2 | valid tokens/s: 75794.2 | physical tokens/s: 76603.4 | loss tokens/s: 55679.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.915810E-01 | lm-global-supervised-tokens: 1.232471E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.567426E-02 | lm-token-buffer-mib: 2.348883E+02 | global_load_balancing_loss: 9.997872E-01 | loss scale: 1.0 | grad norm: 0.217 | params norm: 11147.339 | params rms: 0.060359 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:35:18.688410 | finish at 2026-10-17 19:23:19 + [2026-10-17 05:51:40] iteration 2164/ 2384 | consumed samples: 69248 | elapsed time per iteration (ms): 220382.2 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 76127.7 | valid tokens/s: 76127.7 | physical tokens/s: 77212.2 | loss tokens/s: 56110.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.219574E-01 | lm-global-supervised-tokens: 1.236578E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.441973E-02 | lm-token-buffer-mib: 2.329455E+02 | global_load_balancing_loss: 9.991992E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11147.303 | params rms: 0.060359 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:28:04.087253 | finish at 2026-10-17 19:19:45 + [2026-10-17 05:55:24] iteration 2165/ 2384 | consumed samples: 69280 | elapsed time per iteration (ms): 223222.2 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 75159.1 | valid tokens/s: 75159.1 | physical tokens/s: 75661.9 | loss tokens/s: 56242.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.211272E-01 | lm-global-supervised-tokens: 1.255463E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.547902E-02 | lm-token-buffer-mib: 2.335716E+02 | global_load_balancing_loss: 1.001608E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11147.266 | params rms: 0.060359 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:34:45.653584 | finish at 2026-10-17 19:30:09 + [2026-10-17 05:59:07] iteration 2166/ 2384 | consumed samples: 69312 | elapsed time per iteration (ms): 223471.2 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.65% | FLOPs estimator: workload-aware | logical tokens/s: 75075.5 | valid tokens/s: 75075.5 | physical tokens/s: 75862.2 | loss tokens/s: 57569.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.902965E-01 | lm-global-supervised-tokens: 1.286506E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.319472E-02 | lm-token-buffer-mib: 2.356621E+02 | global_load_balancing_loss: 1.002343E+00 | loss scale: 1.0 | grad norm: 0.276 | params norm: 11147.232 | params rms: 0.060359 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:31:56.712190 | finish at 2026-10-17 19:31:04 + [2026-10-17 06:02:48] iteration 2167/ 2384 | consumed samples: 69344 | elapsed time per iteration (ms): 220964.9 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 75927.0 | valid tokens/s: 75927.0 | physical tokens/s: 76435.1 | loss tokens/s: 51332.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.228633E-01 | lm-global-supervised-tokens: 1.134261E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.297346E-02 | lm-token-buffer-mib: 2.330274E+02 | global_load_balancing_loss: 1.001119E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11147.200 | params rms: 0.060358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:19:09.393703 | finish at 2026-10-17 19:21:57 + [2026-10-17 06:06:33] iteration 2168/ 2384 | consumed samples: 69376 | elapsed time per iteration (ms): 225230.0 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 74489.2 | valid tokens/s: 74489.2 | physical tokens/s: 75159.9 | loss tokens/s: 57665.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.874243E-01 | lm-global-supervised-tokens: 1.298796E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.792341E-02 | lm-token-buffer-mib: 2.342571E+02 | global_load_balancing_loss: 9.998615E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11147.165 | params rms: 0.060358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:30:49.685257 | finish at 2026-10-17 19:37:23 + [2026-10-17 06:10:15] iteration 2169/ 2384 | consumed samples: 69408 | elapsed time per iteration (ms): 221782.7 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 75647.0 | valid tokens/s: 75647.0 | physical tokens/s: 76310.2 | loss tokens/s: 51744.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.094441E-01 | lm-global-supervised-tokens: 1.147600E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.004839E-02 | lm-token-buffer-mib: 2.326332E+02 | global_load_balancing_loss: 1.000882E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11147.131 | params rms: 0.060358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:14:43.290765 | finish at 2026-10-17 19:24:58 + [2026-10-17 06:14:19] iteration 2170/ 2384 | consumed samples: 69440 | elapsed time per iteration (ms): 244144.6 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 68718.3 | valid tokens/s: 68718.3 | physical tokens/s: 69043.7 | loss tokens/s: 51874.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.617446E-01 | lm-global-supervised-tokens: 1.266484E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.171426E-02 | lm-token-buffer-mib: 2.334626E+02 | global_load_balancing_loss: 9.997509E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11147.099 | params rms: 0.060358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:30:46.939688 | finish at 2026-10-17 20:45:06 + [2026-10-17 06:18:02] iteration 2171/ 2384 | consumed samples: 69472 | elapsed time per iteration (ms): 223237.0 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75154.3 | valid tokens/s: 75154.2 | physical tokens/s: 75844.9 | loss tokens/s: 58839.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.617413E-01 | lm-global-supervised-tokens: 1.313510E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.471180E-02 | lm-token-buffer-mib: 2.345193E+02 | global_load_balancing_loss: 9.996725E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11147.065 | params rms: 0.060358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:12:29.488717 | finish at 2026-10-17 19:30:32 + [2026-10-17 06:21:48] iteration 2172/ 2384 | consumed samples: 69504 | elapsed time per iteration (ms): 225586.9 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 74371.4 | valid tokens/s: 74371.4 | physical tokens/s: 74951.6 | loss tokens/s: 56855.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.479285E-01 | lm-global-supervised-tokens: 1.282592E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.524851E-02 | lm-token-buffer-mib: 2.341304E+02 | global_load_balancing_loss: 9.992290E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11147.034 | params rms: 0.060358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:17:04.416178 | finish at 2026-10-17 19:38:52 + [2026-10-17 06:25:31] iteration 2173/ 2384 | consumed samples: 69536 | elapsed time per iteration (ms): 222811.1 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75297.7 | valid tokens/s: 75297.7 | physical tokens/s: 76103.1 | loss tokens/s: 59273.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.347541E-01 | lm-global-supervised-tokens: 1.320686E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.776005E-02 | lm-token-buffer-mib: 2.348061E+02 | global_load_balancing_loss: 1.000738E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11147.003 | params rms: 0.060357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:03:33.139083 | finish at 2026-10-17 19:29:04 + [2026-10-17 06:29:15] iteration 2174/ 2384 | consumed samples: 69568 | elapsed time per iteration (ms): 223817.0 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 74959.5 | valid tokens/s: 74959.5 | physical tokens/s: 75612.9 | loss tokens/s: 60259.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.150890E-01 | lm-global-supervised-tokens: 1.348700E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.223668E-02 | lm-token-buffer-mib: 2.345727E+02 | global_load_balancing_loss: 1.000470E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11146.973 | params rms: 0.060357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:03:21.567478 | finish at 2026-10-17 19:32:36 + [2026-10-17 06:33:01] iteration 2175/ 2384 | consumed samples: 69600 | elapsed time per iteration (ms): 226088.7 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.94% | FLOPs estimator: workload-aware | logical tokens/s: 74206.1 | valid tokens/s: 74206.1 | physical tokens/s: 74717.3 | loss tokens/s: 58073.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.228748E-01 | lm-global-supervised-tokens: 1.312986E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.809041E-02 | lm-token-buffer-mib: 2.328301E+02 | global_load_balancing_loss: 1.000132E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11146.940 | params rms: 0.060357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:07:32.547032 | finish at 2026-10-17 19:40:33 + [2026-10-17 06:36:42] iteration 2176/ 2384 | consumed samples: 69632 | elapsed time per iteration (ms): 220766.4 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 75995.3 | valid tokens/s: 75995.3 | physical tokens/s: 76502.1 | loss tokens/s: 56596.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.809907E-01 | lm-global-supervised-tokens: 1.249450E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.287628E-02 | lm-token-buffer-mib: 2.326867E+02 | global_load_balancing_loss: 9.997790E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11146.906 | params rms: 0.060357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:45:19.407452 | finish at 2026-10-17 19:22:01 + [2026-10-17 06:40:23] iteration 2177/ 2384 | consumed samples: 69664 | elapsed time per iteration (ms): 221097.4 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 75881.4 | valid tokens/s: 75881.4 | physical tokens/s: 76340.4 | loss tokens/s: 59404.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.077760E-01 | lm-global-supervised-tokens: 1.313426E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.195105E-02 | lm-token-buffer-mib: 2.332655E+02 | global_load_balancing_loss: 1.001020E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11146.873 | params rms: 0.060357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:42:47.154091 | finish at 2026-10-17 19:23:10 + [2026-10-17 06:44:07] iteration 2178/ 2384 | consumed samples: 69696 | elapsed time per iteration (ms): 224555.4 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 74713.0 | valid tokens/s: 74713.0 | physical tokens/s: 75849.1 | loss tokens/s: 53720.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.199715E-01 | lm-global-supervised-tokens: 1.206315E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.816718E-02 | lm-token-buffer-mib: 2.356491E+02 | global_load_balancing_loss: 1.001381E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11146.842 | params rms: 0.060357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:50:58.406878 | finish at 2026-10-17 19:35:06 + [2026-10-17 06:47:49] iteration 2179/ 2384 | consumed samples: 69728 | elapsed time per iteration (ms): 221984.0 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.52% | FLOPs estimator: workload-aware | logical tokens/s: 75578.3 | valid tokens/s: 75578.3 | physical tokens/s: 76333.3 | loss tokens/s: 58860.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.725064E-01 | lm-global-supervised-tokens: 1.306618E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.256739E-02 | lm-token-buffer-mib: 2.350962E+02 | global_load_balancing_loss: 1.001566E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11146.813 | params rms: 0.060356 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:38:26.714177 | finish at 2026-10-17 19:26:16 + [2026-10-17 06:51:51] iteration 2180/ 2384 | consumed samples: 69760 | elapsed time per iteration (ms): 241338.0 | throughput per GPU (TFLOP/s/GPU): 18.5 | MFU 5.94% | FLOPs estimator: workload-aware | logical tokens/s: 69517.4 | valid tokens/s: 69517.4 | physical tokens/s: 70312.3 | loss tokens/s: 53064.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.966886E-01 | lm-global-supervised-tokens: 1.280653E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.587484E-02 | lm-token-buffer-mib: 2.327517E+02 | global_load_balancing_loss: 1.001271E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11146.781 | params rms: 0.060356 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:40:32.956438 | finish at 2026-10-17 20:32:23 + [2026-10-17 06:55:30] iteration 2181/ 2384 | consumed samples: 69792 | elapsed time per iteration (ms): 219642.2 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 76384.2 | valid tokens/s: 76384.2 | physical tokens/s: 76924.0 | loss tokens/s: 55902.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.251388E-01 | lm-global-supervised-tokens: 1.227857E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.987297E-02 | lm-token-buffer-mib: 2.338799E+02 | global_load_balancing_loss: 1.000421E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11146.748 | params rms: 0.060356 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:23:07.358177 | finish at 2026-10-17 19:18:38 + [2026-10-17 06:59:13] iteration 2182/ 2384 | consumed samples: 69824 | elapsed time per iteration (ms): 223237.2 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75154.2 | valid tokens/s: 75154.2 | physical tokens/s: 76333.6 | loss tokens/s: 53936.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.126949E-01 | lm-global-supervised-tokens: 1.204052E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.296862E-02 | lm-token-buffer-mib: 2.339314E+02 | global_load_balancing_loss: 9.993678E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11146.716 | params rms: 0.060356 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:31:33.920665 | finish at 2026-10-17 19:30:47 + [2026-10-17 07:02:55] iteration 2183/ 2384 | consumed samples: 69856 | elapsed time per iteration (ms): 221328.9 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 75802.1 | valid tokens/s: 75802.1 | physical tokens/s: 76667.9 | loss tokens/s: 52246.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.635703E-01 | lm-global-supervised-tokens: 1.156362E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.654268E-02 | lm-token-buffer-mib: 2.332247E+02 | global_load_balancing_loss: 9.995003E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11146.682 | params rms: 0.060356 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:21:27.114095 | finish at 2026-10-17 19:24:22 + [2026-10-17 07:06:37] iteration 2184/ 2384 | consumed samples: 69888 | elapsed time per iteration (ms): 222390.4 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 75440.3 | valid tokens/s: 75440.3 | physical tokens/s: 75936.2 | loss tokens/s: 56289.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.616614E-01 | lm-global-supervised-tokens: 1.251828E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.604072E-02 | lm-token-buffer-mib: 2.342942E+02 | global_load_balancing_loss: 9.996681E-01 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11146.648 | params rms: 0.060355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:21:18.082275 | finish at 2026-10-17 19:27:55 + [2026-10-17 07:10:24] iteration 2185/ 2384 | consumed samples: 69920 | elapsed time per iteration (ms): 227182.1 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 73849.0 | valid tokens/s: 73849.0 | physical tokens/s: 74730.1 | loss tokens/s: 53776.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.197073E-01 | lm-global-supervised-tokens: 1.221709E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.233675E-02 | lm-token-buffer-mib: 2.369309E+02 | global_load_balancing_loss: 9.997460E-01 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11146.615 | params rms: 0.060355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:33:29.230510 | finish at 2026-10-17 19:43:54 + [2026-10-17 07:14:09] iteration 2186/ 2384 | consumed samples: 69952 | elapsed time per iteration (ms): 224221.7 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 6.00% | FLOPs estimator: workload-aware | logical tokens/s: 74824.2 | valid tokens/s: 74824.2 | physical tokens/s: 75338.9 | loss tokens/s: 55416.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.246865E-01 | lm-global-supervised-tokens: 1.242550E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.920677E-02 | lm-token-buffer-mib: 2.336578E+02 | global_load_balancing_loss: 9.989558E-01 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11146.585 | params rms: 0.060355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:19:55.892625 | finish at 2026-10-17 19:34:04 + [2026-10-17 07:17:50] iteration 2187/ 2384 | consumed samples: 69984 | elapsed time per iteration (ms): 221949.4 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.07% | FLOPs estimator: workload-aware | logical tokens/s: 75590.1 | valid tokens/s: 75590.1 | physical tokens/s: 76035.5 | loss tokens/s: 62883.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.109573E-01 | lm-global-supervised-tokens: 1.395691E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.585573E-02 | lm-token-buffer-mib: 2.328819E+02 | global_load_balancing_loss: 1.000284E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11146.551 | params rms: 0.060355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:08:44.037802 | finish at 2026-10-17 19:26:35 + [2026-10-17 07:21:31] iteration 2188/ 2384 | consumed samples: 70016 | elapsed time per iteration (ms): 220326.0 | throughput per GPU (TFLOP/s/GPU): 26.5 | MFU 8.51% | FLOPs estimator: workload-aware | logical tokens/s: 76147.2 | valid tokens/s: 76147.2 | physical tokens/s: 76379.3 | loss tokens/s: 57214.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.383259E-01 | lm-global-supervised-tokens: 1.260586E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.255203E-02 | lm-token-buffer-mib: 2.324343E+02 | global_load_balancing_loss: 1.001011E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11146.516 | params rms: 0.060355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:59:43.896883 | finish at 2026-10-17 19:21:15 + [2026-10-17 07:25:16] iteration 2189/ 2384 | consumed samples: 70048 | elapsed time per iteration (ms): 225530.1 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.45% | FLOPs estimator: workload-aware | logical tokens/s: 74390.1 | valid tokens/s: 74390.1 | physical tokens/s: 75679.1 | loss tokens/s: 56139.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.049320E-01 | lm-global-supervised-tokens: 1.266123E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.640046E-02 | lm-token-buffer-mib: 2.367508E+02 | global_load_balancing_loss: 1.001908E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11146.485 | params rms: 0.060355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:12:58.363338 | finish at 2026-10-17 19:38:15 + [2026-10-17 07:29:23] iteration 2190/ 2384 | consumed samples: 70080 | elapsed time per iteration (ms): 246418.8 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 68084.0 | valid tokens/s: 68084.0 | physical tokens/s: 69162.8 | loss tokens/s: 53289.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.178916E-01 | lm-global-supervised-tokens: 1.313153E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.822988E-02 | lm-token-buffer-mib: 2.339647E+02 | global_load_balancing_loss: 1.002005E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11146.452 | params rms: 0.060354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:16:45.251940 | finish at 2026-10-17 20:46:08 + [2026-10-17 07:33:07] iteration 2191/ 2384 | consumed samples: 70112 | elapsed time per iteration (ms): 223844.8 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 74950.1 | valid tokens/s: 74950.0 | physical tokens/s: 75430.6 | loss tokens/s: 52821.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.316096E-01 | lm-global-supervised-tokens: 1.182369E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.369032E-02 | lm-token-buffer-mib: 2.338234E+02 | global_load_balancing_loss: 1.000518E+00 | loss scale: 1.0 | grad norm: 0.354 | params norm: 11146.418 | params rms: 0.060354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:00:02.037564 | finish at 2026-10-17 19:33:09 + [2026-10-17 07:36:47] iteration 2192/ 2384 | consumed samples: 70144 | elapsed time per iteration (ms): 220404.7 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 76120.0 | valid tokens/s: 76120.0 | physical tokens/s: 76590.7 | loss tokens/s: 55091.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.661346E-01 | lm-global-supervised-tokens: 1.214252E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.661817E-02 | lm-token-buffer-mib: 2.334734E+02 | global_load_balancing_loss: 1.000105E+00 | loss scale: 1.0 | grad norm: 0.207 | params norm: 11146.384 | params rms: 0.060354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:45:17.707901 | finish at 2026-10-17 19:22:05 + [2026-10-17 07:40:29] iteration 2193/ 2384 | consumed samples: 70176 | elapsed time per iteration (ms): 222132.1 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 75528.0 | valid tokens/s: 75528.0 | physical tokens/s: 76242.1 | loss tokens/s: 58955.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.740405E-01 | lm-global-supervised-tokens: 1.309586E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.216126E-02 | lm-token-buffer-mib: 2.351270E+02 | global_load_balancing_loss: 1.000752E+00 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11146.351 | params rms: 0.060354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:47:07.222696 | finish at 2026-10-17 19:27:36 + [2026-10-17 07:44:08] iteration 2194/ 2384 | consumed samples: 70208 | elapsed time per iteration (ms): 219242.5 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 76523.4 | valid tokens/s: 76523.4 | physical tokens/s: 77016.0 | loss tokens/s: 59156.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.240109E-01 | lm-global-supervised-tokens: 1.296967E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.871151E-02 | lm-token-buffer-mib: 2.335391E+02 | global_load_balancing_loss: 1.001091E+00 | loss scale: 1.0 | grad norm: 0.235 | params norm: 11146.318 | params rms: 0.060354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:34:16.078365 | finish at 2026-10-17 19:18:24 + [2026-10-17 07:47:51] iteration 2195/ 2384 | consumed samples: 70240 | elapsed time per iteration (ms): 222999.7 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 75234.1 | valid tokens/s: 75234.1 | physical tokens/s: 75660.4 | loss tokens/s: 54719.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.608677E-01 | lm-global-supervised-tokens: 1.220234E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.536903E-02 | lm-token-buffer-mib: 2.336269E+02 | global_load_balancing_loss: 9.995317E-01 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11146.289 | params rms: 0.060354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:42:26.942187 | finish at 2026-10-17 19:30:18 + [2026-10-17 07:51:33] iteration 2196/ 2384 | consumed samples: 70272 | elapsed time per iteration (ms): 221951.6 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.81% | FLOPs estimator: workload-aware | logical tokens/s: 75589.4 | valid tokens/s: 75589.4 | physical tokens/s: 76422.0 | loss tokens/s: 62848.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.764462E-01 | lm-global-supervised-tokens: 1.394940E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.251327E-02 | lm-token-buffer-mib: 2.350287E+02 | global_load_balancing_loss: 1.000845E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11146.257 | params rms: 0.060353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:35:26.906507 | finish at 2026-10-17 19:27:00 + [2026-10-17 07:55:15] iteration 2197/ 2384 | consumed samples: 70304 | elapsed time per iteration (ms): 221781.5 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75647.4 | valid tokens/s: 75647.4 | physical tokens/s: 76794.6 | loss tokens/s: 59224.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.212556E-01 | lm-global-supervised-tokens: 1.313479E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.884400E-02 | lm-token-buffer-mib: 2.345392E+02 | global_load_balancing_loss: 1.000305E+00 | loss scale: 1.0 | grad norm: 0.288 | params norm: 11146.226 | params rms: 0.060353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:31:13.146984 | finish at 2026-10-17 19:26:28 + [2026-10-17 07:59:00] iteration 2198/ 2384 | consumed samples: 70336 | elapsed time per iteration (ms): 224392.7 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 74767.2 | valid tokens/s: 74767.2 | physical tokens/s: 75297.7 | loss tokens/s: 55198.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.265125E-01 | lm-global-supervised-tokens: 1.238605E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.622158E-02 | lm-token-buffer-mib: 2.334822E+02 | global_load_balancing_loss: 1.001045E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11146.194 | params rms: 0.060353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:35:37.044010 | finish at 2026-10-17 19:34:37 + [2026-10-17 08:02:44] iteration 2199/ 2384 | consumed samples: 70368 | elapsed time per iteration (ms): 224495.8 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 74732.8 | valid tokens/s: 74732.8 | physical tokens/s: 75618.9 | loss tokens/s: 60589.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.406842E-01 | lm-global-supervised-tokens: 1.360215E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.507245E-02 | lm-token-buffer-mib: 2.350757E+02 | global_load_balancing_loss: 9.995400E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11146.162 | params rms: 0.060353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:32:11.728384 | finish at 2026-10-17 19:34:56 + [2026-10-17 08:06:47] iteration 2200/ 2384 | consumed samples: 70400 | elapsed time per iteration (ms): 242917.0 | throughput per GPU (TFLOP/s/GPU): 18.4 | MFU 5.90% | FLOPs estimator: workload-aware | logical tokens/s: 69065.5 | valid tokens/s: 69065.5 | physical tokens/s: 69648.1 | loss tokens/s: 53025.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.898910E-01 | lm-global-supervised-tokens: 1.288089E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.350166E-02 | lm-token-buffer-mib: 2.342648E+02 | global_load_balancing_loss: 9.993525E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11146.129 | params rms: 0.060353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:24:56.733099 | finish at 2026-10-17 20:31:44 +times across ranks (ms): + save-checkpoint: + rank 0: 25276.38 + rank 1: 25276.38 + rank 2: 25276.42 + rank 3: 25276.41 + rank 4: 25276.37 + rank 5: 25276.41 + rank 6: 25276.41 + rank 7: 25276.40 + rank 8: 25276.92 + rank 9: 25276.96 + rank 10: 25276.97 + rank 11: 25276.91 + rank 12: 25276.96 + rank 13: 25276.92 + rank 14: 25276.94 + rank 15: 25276.93 + rank 16: 25276.42 + rank 17: 25276.56 + rank 18: 25276.59 + rank 19: 25276.56 + rank 20: 25276.54 + rank 21: 25276.53 + rank 22: 25276.54 + rank 23: 25276.54 + rank 24: 25276.34 + rank 25: 25276.32 + rank 26: 25276.31 + rank 27: 25276.30 + rank 28: 25276.32 + rank 29: 25276.62 + rank 30: 25276.33 + rank 31: 25276.23 + rank 32: 25279.29 + rank 33: 25279.32 + rank 34: 25279.33 + rank 35: 25279.25 + rank 36: 25279.09 + rank 37: 25279.08 + rank 38: 25279.21 + rank 39: 25279.19 + rank 40: 25277.04 + rank 41: 25277.03 + rank 42: 25277.03 + rank 43: 25277.03 + rank 44: 25277.01 + rank 45: 25277.05 + rank 46: 25277.01 + rank 47: 25277.01 + rank 48: 25276.92 + rank 49: 25276.76 + rank 50: 25276.94 + rank 51: 25276.90 + rank 52: 25276.87 + rank 53: 25276.87 + rank 54: 25276.87 + rank 55: 25276.88 + rank 56: 25276.98 + rank 57: 25277.01 + rank 58: 25277.01 + rank 59: 25277.01 + rank 60: 25276.99 + rank 61: 25276.98 + rank 62: 25276.99 + rank 63: 25276.96 + [2026-10-17 08:10:52] iteration 2201/ 2384 | consumed samples: 70432 | elapsed time per iteration (ms): 219237.1 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 76525.4 | valid tokens/s: 76525.4 | physical tokens/s: 77003.3 | loss tokens/s: 59764.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.449117E-01 | lm-global-supervised-tokens: 1.310250E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.084830E-01 | lm-token-buffer-mib: 2.333181E+02 | global_load_balancing_loss: 1.000525E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11146.091 | params rms: 0.060352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:08:40.389366 | finish at 2026-10-17 19:19:32 + [2026-10-17 08:14:39] iteration 2202/ 2384 | consumed samples: 70464 | elapsed time per iteration (ms): 227016.0 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 73903.1 | valid tokens/s: 73903.1 | physical tokens/s: 74753.5 | loss tokens/s: 51346.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.453139E-01 | lm-global-supervised-tokens: 1.165642E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.063537E-02 | lm-token-buffer-mib: 2.343187E+02 | global_load_balancing_loss: 1.001352E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11146.053 | params rms: 0.060352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:28:36.907927 | finish at 2026-10-17 19:43:15 + [2026-10-17 08:18:19] iteration 2203/ 2384 | consumed samples: 70496 | elapsed time per iteration (ms): 220755.7 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 75999.0 | valid tokens/s: 75999.0 | physical tokens/s: 76878.6 | loss tokens/s: 58949.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.118364E-01 | lm-global-supervised-tokens: 1.301345E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.595759E-02 | lm-token-buffer-mib: 2.343263E+02 | global_load_balancing_loss: 9.995444E-01 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11146.017 | params rms: 0.060352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:05:56.773046 | finish at 2026-10-17 19:24:16 + [2026-10-17 08:21:59] iteration 2204/ 2384 | consumed samples: 70528 | elapsed time per iteration (ms): 219498.4 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 76434.3 | valid tokens/s: 76434.3 | physical tokens/s: 76939.9 | loss tokens/s: 58211.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.645620E-01 | lm-global-supervised-tokens: 1.277742E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.197493E-02 | lm-token-buffer-mib: 2.331721E+02 | global_load_balancing_loss: 1.001683E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11145.982 | params rms: 0.060352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:58:29.705558 | finish at 2026-10-17 19:20:29 + [2026-10-17 08:25:41] iteration 2205/ 2384 | consumed samples: 70560 | elapsed time per iteration (ms): 222135.5 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 75526.8 | valid tokens/s: 75526.8 | physical tokens/s: 75884.8 | loss tokens/s: 56901.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.625407E-01 | lm-global-supervised-tokens: 1.263989E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.548401E-02 | lm-token-buffer-mib: 2.325375E+02 | global_load_balancing_loss: 1.000191E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11145.947 | params rms: 0.060352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:02:42.254193 | finish at 2026-10-17 19:28:23 + [2026-10-17 08:29:27] iteration 2206/ 2384 | consumed samples: 70592 | elapsed time per iteration (ms): 225816.4 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.23% | FLOPs estimator: workload-aware | logical tokens/s: 74295.7 | valid tokens/s: 74295.7 | physical tokens/s: 74524.0 | loss tokens/s: 51439.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.448770E-01 | lm-global-supervised-tokens: 1.161578E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.942748E-02 | lm-token-buffer-mib: 2.322808E+02 | global_load_balancing_loss: 1.000743E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11145.912 | params rms: 0.060351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:09:55.319718 | finish at 2026-10-17 19:39:22 + [2026-10-17 08:33:11] iteration 2207/ 2384 | consumed samples: 70624 | elapsed time per iteration (ms): 223917.6 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 74925.7 | valid tokens/s: 74925.7 | physical tokens/s: 75706.7 | loss tokens/s: 54734.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.301755E-01 | lm-global-supervised-tokens: 1.225600E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.396422E-02 | lm-token-buffer-mib: 2.352497E+02 | global_load_balancing_loss: 9.995894E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11145.875 | params rms: 0.060351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:00:33.416704 | finish at 2026-10-17 19:33:44 + [2026-10-17 08:36:54] iteration 2208/ 2384 | consumed samples: 70656 | elapsed time per iteration (ms): 222973.0 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75243.1 | valid tokens/s: 75243.1 | physical tokens/s: 76020.2 | loss tokens/s: 56927.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.608947E-01 | lm-global-supervised-tokens: 1.269319E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.163054E-02 | lm-token-buffer-mib: 2.339449E+02 | global_load_balancing_loss: 9.995162E-01 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11145.841 | params rms: 0.060351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:54:03.255898 | finish at 2026-10-17 19:30:57 + [2026-10-17 08:40:37] iteration 2209/ 2384 | consumed samples: 70688 | elapsed time per iteration (ms): 223101.5 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 75199.7 | valid tokens/s: 75199.7 | physical tokens/s: 76024.4 | loss tokens/s: 56718.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.923994E-01 | lm-global-supervised-tokens: 1.265399E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.539825E-02 | lm-token-buffer-mib: 2.361932E+02 | global_load_balancing_loss: 9.998859E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11145.806 | params rms: 0.060351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:50:42.770725 | finish at 2026-10-17 19:31:20 + [2026-10-17 08:44:42] iteration 2210/ 2384 | consumed samples: 70720 | elapsed time per iteration (ms): 244811.9 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 68530.8 | valid tokens/s: 68530.8 | physical tokens/s: 69183.3 | loss tokens/s: 51523.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.806707E-01 | lm-global-supervised-tokens: 1.261361E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.441792E-02 | lm-token-buffer-mib: 2.348061E+02 | global_load_balancing_loss: 1.000821E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11145.769 | params rms: 0.060351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:49:57.276971 | finish at 2026-10-17 20:34:39 + [2026-10-17 08:48:23] iteration 2211/ 2384 | consumed samples: 70752 | elapsed time per iteration (ms): 221517.5 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.02% | FLOPs estimator: workload-aware | logical tokens/s: 75737.5 | valid tokens/s: 75737.5 | physical tokens/s: 76659.0 | loss tokens/s: 59529.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.800322E-01 | lm-global-supervised-tokens: 1.318690E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.222995E-02 | lm-token-buffer-mib: 2.360871E+02 | global_load_balancing_loss: 1.000843E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11145.730 | params rms: 0.060351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:38:42.522537 | finish at 2026-10-17 19:27:06 + [2026-10-17 08:52:07] iteration 2212/ 2384 | consumed samples: 70784 | elapsed time per iteration (ms): 223594.8 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.72% | FLOPs estimator: workload-aware | logical tokens/s: 75034.0 | valid tokens/s: 75034.0 | physical tokens/s: 75571.3 | loss tokens/s: 53400.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.011872E-01 | lm-global-supervised-tokens: 1.194016E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.884196E-02 | lm-token-buffer-mib: 2.326820E+02 | global_load_balancing_loss: 1.000976E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11145.696 | params rms: 0.060350 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:40:58.299284 | finish at 2026-10-17 19:33:05 + [2026-10-17 08:55:50] iteration 2213/ 2384 | consumed samples: 70816 | elapsed time per iteration (ms): 222820.1 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 75294.9 | valid tokens/s: 75294.9 | physical tokens/s: 75918.8 | loss tokens/s: 59981.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.778110E-01 | lm-global-supervised-tokens: 1.336497E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.190508E-02 | lm-token-buffer-mib: 2.339304E+02 | global_load_balancing_loss: 1.000618E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11145.664 | params rms: 0.060350 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:35:02.230385 | finish at 2026-10-17 19:30:52 + [2026-10-17 08:59:34] iteration 2214/ 2384 | consumed samples: 70848 | elapsed time per iteration (ms): 224507.6 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 74728.8 | valid tokens/s: 74728.8 | physical tokens/s: 75586.7 | loss tokens/s: 52797.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.578797E-01 | lm-global-supervised-tokens: 1.185333E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.513461E-02 | lm-token-buffer-mib: 2.327751E+02 | global_load_balancing_loss: 1.000816E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11145.633 | params rms: 0.060350 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:36:06.296389 | finish at 2026-10-17 19:35:40 + [2026-10-17 09:03:18] iteration 2215/ 2384 | consumed samples: 70880 | elapsed time per iteration (ms): 224318.6 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 74791.8 | valid tokens/s: 74791.8 | physical tokens/s: 76033.6 | loss tokens/s: 53847.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.163101E-01 | lm-global-supervised-tokens: 1.207902E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.176419E-02 | lm-token-buffer-mib: 2.342715E+02 | global_load_balancing_loss: 1.000369E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11145.600 | params rms: 0.060350 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:31:49.846774 | finish at 2026-10-17 19:35:08 + [2026-10-17 09:07:02] iteration 2216/ 2384 | consumed samples: 70912 | elapsed time per iteration (ms): 223751.3 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.42% | FLOPs estimator: workload-aware | logical tokens/s: 74981.2 | valid tokens/s: 74981.2 | physical tokens/s: 75616.5 | loss tokens/s: 56555.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.056816E-01 | lm-global-supervised-tokens: 1.265437E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.127808E-02 | lm-token-buffer-mib: 2.342826E+02 | global_load_balancing_loss: 9.997187E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11145.566 | params rms: 0.060350 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:26:30.211967 | finish at 2026-10-17 19:33:32 + [2026-10-17 09:10:42] iteration 2217/ 2384 | consumed samples: 70944 | elapsed time per iteration (ms): 220403.7 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 76120.4 | valid tokens/s: 76120.4 | physical tokens/s: 76462.2 | loss tokens/s: 59200.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.662326E-01 | lm-global-supervised-tokens: 1.304790E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.381846E-02 | lm-token-buffer-mib: 2.330317E+02 | global_load_balancing_loss: 9.990876E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11145.532 | params rms: 0.060349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:13:27.414614 | finish at 2026-10-17 19:24:10 + [2026-10-17 09:14:24] iteration 2218/ 2384 | consumed samples: 70976 | elapsed time per iteration (ms): 221791.4 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 75644.0 | valid tokens/s: 75644.0 | physical tokens/s: 76471.1 | loss tokens/s: 55501.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.906330E-01 | lm-global-supervised-tokens: 1.230981E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.631985E-02 | lm-token-buffer-mib: 2.329541E+02 | global_load_balancing_loss: 1.000857E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11145.501 | params rms: 0.060349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:13:37.369781 | finish at 2026-10-17 19:28:02 + [2026-10-17 09:18:07] iteration 2219/ 2384 | consumed samples: 71008 | elapsed time per iteration (ms): 223125.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 75191.8 | valid tokens/s: 75191.8 | physical tokens/s: 76110.3 | loss tokens/s: 59783.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.885454E-01 | lm-global-supervised-tokens: 1.333924E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.330875E-02 | lm-token-buffer-mib: 2.328937E+02 | global_load_balancing_loss: 1.000396E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11145.470 | params rms: 0.060349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:13:35.700846 | finish at 2026-10-17 19:31:43 + [2026-10-17 09:22:09] iteration 2220/ 2384 | consumed samples: 71040 | elapsed time per iteration (ms): 241913.1 | throughput per GPU (TFLOP/s/GPU): 18.1 | MFU 5.81% | FLOPs estimator: workload-aware | logical tokens/s: 69352.1 | valid tokens/s: 69352.1 | physical tokens/s: 70049.1 | loss tokens/s: 53209.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.611268E-01 | lm-global-supervised-tokens: 1.287217E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.593847E-02 | lm-token-buffer-mib: 2.344863E+02 | global_load_balancing_loss: 1.001280E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11145.438 | params rms: 0.060349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:01:13.745859 | finish at 2026-10-17 20:23:23 + [2026-10-17 09:25:54] iteration 2221/ 2384 | consumed samples: 71072 | elapsed time per iteration (ms): 224847.5 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 74615.9 | valid tokens/s: 74615.9 | physical tokens/s: 74910.9 | loss tokens/s: 55239.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.257086E-01 | lm-global-supervised-tokens: 1.242041E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.245752E-02 | lm-token-buffer-mib: 2.324553E+02 | global_load_balancing_loss: 1.001454E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11145.406 | params rms: 0.060349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:10:50.140687 | finish at 2026-10-17 19:36:44 + [2026-10-17 09:29:36] iteration 2222/ 2384 | consumed samples: 71104 | elapsed time per iteration (ms): 221530.7 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 75733.1 | valid tokens/s: 75733.1 | physical tokens/s: 76523.6 | loss tokens/s: 54724.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.148356E-01 | lm-global-supervised-tokens: 1.212306E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.164497E-02 | lm-token-buffer-mib: 2.333749E+02 | global_load_balancing_loss: 9.990492E-01 | loss scale: 1.0 | grad norm: 0.352 | params norm: 11145.374 | params rms: 0.060349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:58:07.975751 | finish at 2026-10-17 19:27:44 + [2026-10-17 09:33:21] iteration 2223/ 2384 | consumed samples: 71136 | elapsed time per iteration (ms): 225204.8 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 74497.6 | valid tokens/s: 74497.6 | physical tokens/s: 76344.5 | loss tokens/s: 56847.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.541861E-01 | lm-global-supervised-tokens: 1.280237E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.177015E-02 | lm-token-buffer-mib: 2.374410E+02 | global_load_balancing_loss: 9.997643E-01 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11145.339 | params rms: 0.060348 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:04:17.979154 | finish at 2026-10-17 19:37:39 + [2026-10-17 09:37:01] iteration 2224/ 2384 | consumed samples: 71168 | elapsed time per iteration (ms): 220400.5 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 76121.3 | valid tokens/s: 76121.3 | physical tokens/s: 76713.0 | loss tokens/s: 59029.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.920615E-01 | lm-global-supervised-tokens: 1.301007E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.334961E-02 | lm-token-buffer-mib: 2.328692E+02 | global_load_balancing_loss: 1.000455E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11145.307 | params rms: 0.060348 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:47:44.087868 | finish at 2026-10-17 19:24:45 + [2026-10-17 09:40:46] iteration 2225/ 2384 | consumed samples: 71200 | elapsed time per iteration (ms): 224545.8 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 74716.1 | valid tokens/s: 74716.1 | physical tokens/s: 75560.2 | loss tokens/s: 58867.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.053131E-01 | lm-global-supervised-tokens: 1.321836E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.177156E-02 | lm-token-buffer-mib: 2.344593E+02 | global_load_balancing_loss: 9.999942E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11145.274 | params rms: 0.060348 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:55:02.783105 | finish at 2026-10-17 19:35:49 + [2026-10-17 09:44:27] iteration 2226/ 2384 | consumed samples: 71232 | elapsed time per iteration (ms): 221575.0 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.04% | FLOPs estimator: workload-aware | logical tokens/s: 75718.0 | valid tokens/s: 75718.0 | physical tokens/s: 76510.0 | loss tokens/s: 60308.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.465597E-01 | lm-global-supervised-tokens: 1.336284E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.980301E-02 | lm-token-buffer-mib: 2.355543E+02 | global_load_balancing_loss: 1.000370E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11145.236 | params rms: 0.060348 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:43:28.849103 | finish at 2026-10-17 19:27:56 + [2026-10-17 09:48:15] iteration 2227/ 2384 | consumed samples: 71264 | elapsed time per iteration (ms): 227121.0 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 73869.0 | valid tokens/s: 73868.9 | physical tokens/s: 75033.7 | loss tokens/s: 55437.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.426036E-01 | lm-global-supervised-tokens: 1.259090E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.410203E-02 | lm-token-buffer-mib: 2.355953E+02 | global_load_balancing_loss: 1.000809E+00 | loss scale: 1.0 | grad norm: 0.220 | params norm: 11145.201 | params rms: 0.060348 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:54:17.997457 | finish at 2026-10-17 19:42:33 + [2026-10-17 09:51:57] iteration 2228/ 2384 | consumed samples: 71296 | elapsed time per iteration (ms): 222639.6 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75355.8 | valid tokens/s: 75355.8 | physical tokens/s: 75540.5 | loss tokens/s: 51657.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.871469E-01 | lm-global-supervised-tokens: 1.150111E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.573762E-02 | lm-token-buffer-mib: 2.327422E+02 | global_load_balancing_loss: 1.000307E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11145.166 | params rms: 0.060347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:38:51.783966 | finish at 2026-10-17 19:30:49 + [2026-10-17 09:55:41] iteration 2229/ 2384 | consumed samples: 71328 | elapsed time per iteration (ms): 223706.1 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 74996.6 | valid tokens/s: 74996.6 | physical tokens/s: 76489.5 | loss tokens/s: 61197.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.178007E-01 | lm-global-supervised-tokens: 1.369033E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.017903E-02 | lm-token-buffer-mib: 2.350562E+02 | global_load_balancing_loss: 1.001116E+00 | loss scale: 1.0 | grad norm: 0.303 | params norm: 11145.133 | params rms: 0.060347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:37:54.449933 | finish at 2026-10-17 19:33:35 + [2026-10-17 09:59:46] iteration 2230/ 2384 | consumed samples: 71360 | elapsed time per iteration (ms): 244860.3 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 68517.5 | valid tokens/s: 68517.5 | physical tokens/s: 68762.9 | loss tokens/s: 50134.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.822709E-01 | lm-global-supervised-tokens: 1.227592E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.377819E-02 | lm-token-buffer-mib: 2.324221E+02 | global_load_balancing_loss: 1.000670E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11145.102 | params rms: 0.060347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:28:28.489771 | finish at 2026-10-17 20:28:14 + [2026-10-17 10:03:29] iteration 2231/ 2384 | consumed samples: 71392 | elapsed time per iteration (ms): 222882.6 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 75273.8 | valid tokens/s: 75273.8 | physical tokens/s: 76205.0 | loss tokens/s: 56896.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.478009E-01 | lm-global-supervised-tokens: 1.268132E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.422490E-02 | lm-token-buffer-mib: 2.370168E+02 | global_load_balancing_loss: 9.995787E-01 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11145.068 | params rms: 0.060347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:28:21.037373 | finish at 2026-10-17 19:31:50 + [2026-10-17 10:07:11] iteration 2232/ 2384 | consumed samples: 71424 | elapsed time per iteration (ms): 222109.1 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 75535.8 | valid tokens/s: 75535.8 | physical tokens/s: 76376.5 | loss tokens/s: 56947.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.816879E-01 | lm-global-supervised-tokens: 1.264857E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.756125E-02 | lm-token-buffer-mib: 2.350882E+02 | global_load_balancing_loss: 9.988151E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11145.035 | params rms: 0.060347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:22:40.577526 | finish at 2026-10-17 19:29:51 + [2026-10-17 10:10:52] iteration 2233/ 2384 | consumed samples: 71456 | elapsed time per iteration (ms): 221496.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75744.7 | valid tokens/s: 75744.7 | physical tokens/s: 76794.5 | loss tokens/s: 58793.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.372020E-01 | lm-global-supervised-tokens: 1.302245E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.317694E-02 | lm-token-buffer-mib: 2.368417E+02 | global_load_balancing_loss: 9.996849E-01 | loss scale: 1.0 | grad norm: 0.379 | params norm: 11145.000 | params rms: 0.060347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:17:26.002139 | finish at 2026-10-17 19:28:18 + [2026-10-17 10:14:36] iteration 2234/ 2384 | consumed samples: 71488 | elapsed time per iteration (ms): 223572.1 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 75041.5 | valid tokens/s: 75041.5 | physical tokens/s: 76001.5 | loss tokens/s: 53821.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.029502E-01 | lm-global-supervised-tokens: 1.203288E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.153107E-02 | lm-token-buffer-mib: 2.356850E+02 | global_load_balancing_loss: 1.000738E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11144.967 | params rms: 0.060346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:18:55.821140 | finish at 2026-10-17 19:33:32 + [2026-10-17 10:18:21] iteration 2235/ 2384 | consumed samples: 71520 | elapsed time per iteration (ms): 225417.3 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 74427.3 | valid tokens/s: 74427.3 | physical tokens/s: 74689.7 | loss tokens/s: 53382.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.180833E-01 | lm-global-supervised-tokens: 1.203334E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.777310E-02 | lm-token-buffer-mib: 2.332969E+02 | global_load_balancing_loss: 1.000137E+00 | loss scale: 1.0 | grad norm: 0.303 | params norm: 11144.935 | params rms: 0.060346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:19:47.177769 | finish at 2026-10-17 19:38:08 + [2026-10-17 10:22:03] iteration 2236/ 2384 | consumed samples: 71552 | elapsed time per iteration (ms): 221499.1 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.05% | FLOPs estimator: workload-aware | logical tokens/s: 75743.9 | valid tokens/s: 75743.9 | physical tokens/s: 76396.9 | loss tokens/s: 57369.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.636746E-01 | lm-global-supervised-tokens: 1.270720E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.266204E-02 | lm-token-buffer-mib: 2.341833E+02 | global_load_balancing_loss: 1.000844E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11144.908 | params rms: 0.060346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:06:21.869477 | finish at 2026-10-17 19:28:25 + [2026-10-17 10:25:44] iteration 2237/ 2384 | consumed samples: 71584 | elapsed time per iteration (ms): 221583.0 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.03% | FLOPs estimator: workload-aware | logical tokens/s: 75715.3 | valid tokens/s: 75715.3 | physical tokens/s: 76773.0 | loss tokens/s: 60204.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.798818E-01 | lm-global-supervised-tokens: 1.334035E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.231219E-02 | lm-token-buffer-mib: 2.362738E+02 | global_load_balancing_loss: 1.001072E+00 | loss scale: 1.0 | grad norm: 0.349 | params norm: 11144.882 | params rms: 0.060346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:02:52.695595 | finish at 2026-10-17 19:28:37 + [2026-10-17 10:29:27] iteration 2238/ 2384 | consumed samples: 71616 | elapsed time per iteration (ms): 222973.1 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 75243.2 | valid tokens/s: 75243.2 | physical tokens/s: 76099.2 | loss tokens/s: 59857.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.025834E-01 | lm-global-supervised-tokens: 1.334655E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.860772E-02 | lm-token-buffer-mib: 2.355368E+02 | global_load_balancing_loss: 1.001524E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11144.856 | params rms: 0.060346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:02:34.068485 | finish at 2026-10-17 19:32:01 + [2026-10-17 10:33:09] iteration 2239/ 2384 | consumed samples: 71648 | elapsed time per iteration (ms): 221476.8 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 75751.4 | valid tokens/s: 75751.4 | physical tokens/s: 76754.3 | loss tokens/s: 60777.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.661242E-01 | lm-global-supervised-tokens: 1.346074E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.461720E-02 | lm-token-buffer-mib: 2.338425E+02 | global_load_balancing_loss: 9.996938E-01 | loss scale: 1.0 | grad norm: 0.335 | params norm: 11144.828 | params rms: 0.060346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:55:14.142460 | finish at 2026-10-17 19:28:23 + [2026-10-17 10:37:12] iteration 2240/ 2384 | consumed samples: 71680 | elapsed time per iteration (ms): 243223.4 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 68978.5 | valid tokens/s: 68978.5 | physical tokens/s: 69474.1 | loss tokens/s: 51507.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.011317E-01 | lm-global-supervised-tokens: 1.252773E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.204164E-02 | lm-token-buffer-mib: 2.340427E+02 | global_load_balancing_loss: 1.000769E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11144.798 | params rms: 0.060345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:43:44.163712 | finish at 2026-10-17 20:20:56 +times across ranks (ms): + save-checkpoint: + rank 0: 24911.72 + rank 1: 24911.73 + rank 2: 24911.74 + rank 3: 24911.73 + rank 4: 24911.75 + rank 5: 24911.75 + rank 6: 24911.75 + rank 7: 24911.73 + rank 8: 24911.98 + rank 9: 24912.00 + rank 10: 24911.85 + rank 11: 24912.00 + rank 12: 24911.97 + rank 13: 24911.97 + rank 14: 24911.99 + rank 15: 24911.99 + rank 16: 24911.45 + rank 17: 24911.43 + rank 18: 24911.45 + rank 19: 24911.43 + rank 20: 24911.42 + rank 21: 24911.42 + rank 22: 24911.44 + rank 23: 24911.41 + rank 24: 24912.11 + rank 25: 24912.23 + rank 26: 24912.11 + rank 27: 24912.18 + rank 28: 24912.89 + rank 29: 24912.21 + rank 30: 24912.18 + rank 31: 24912.21 + rank 32: 24912.59 + rank 33: 24912.39 + rank 34: 24912.53 + rank 35: 24912.57 + rank 36: 24912.46 + rank 37: 24912.53 + rank 38: 24912.44 + rank 39: 24912.37 + rank 40: 24911.39 + rank 41: 24911.52 + rank 42: 24911.52 + rank 43: 24911.54 + rank 44: 24911.52 + rank 45: 24911.52 + rank 46: 24911.54 + rank 47: 24911.52 + rank 48: 24912.04 + rank 49: 24911.93 + rank 50: 24912.07 + rank 51: 24912.03 + rank 52: 24912.06 + rank 53: 24912.09 + rank 54: 24912.00 + rank 55: 24912.03 + rank 56: 24912.16 + rank 57: 24912.19 + rank 58: 24912.24 + rank 59: 24912.19 + rank 60: 24912.18 + rank 61: 24912.15 + rank 62: 24912.15 + rank 63: 24912.15 + [2026-10-17 10:41:19] iteration 2241/ 2384 | consumed samples: 71712 | elapsed time per iteration (ms): 221477.9 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 75751.2 | valid tokens/s: 75751.2 | physical tokens/s: 76368.2 | loss tokens/s: 55797.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.758264E-01 | lm-global-supervised-tokens: 1.235796E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.532024E-01 | lm-token-buffer-mib: 2.340783E+02 | global_load_balancing_loss: 1.001272E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11144.766 | params rms: 0.060345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:47:51.336226 | finish at 2026-10-17 19:29:10 + [2026-10-17 10:45:06] iteration 2242/ 2384 | consumed samples: 71744 | elapsed time per iteration (ms): 227430.2 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.64% | FLOPs estimator: workload-aware | logical tokens/s: 73768.4 | valid tokens/s: 73768.4 | physical tokens/s: 74562.5 | loss tokens/s: 54871.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.365123E-01 | lm-global-supervised-tokens: 1.247941E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.230253E-02 | lm-token-buffer-mib: 2.338152E+02 | global_load_balancing_loss: 1.001401E+00 | loss scale: 1.0 | grad norm: 0.267 | params norm: 11144.735 | params rms: 0.060345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:58:15.085063 | finish at 2026-10-17 19:43:21 + [2026-10-17 10:48:47] iteration 2243/ 2384 | consumed samples: 71776 | elapsed time per iteration (ms): 220467.3 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 76098.2 | valid tokens/s: 76098.2 | physical tokens/s: 76773.9 | loss tokens/s: 56035.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.895812E-01 | lm-global-supervised-tokens: 1.235409E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.293574E-02 | lm-token-buffer-mib: 2.339805E+02 | global_load_balancing_loss: 9.999702E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11144.704 | params rms: 0.060345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:38:05.895645 | finish at 2026-10-17 19:26:52 + [2026-10-17 10:52:27] iteration 2244/ 2384 | consumed samples: 71808 | elapsed time per iteration (ms): 220791.4 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 75986.6 | valid tokens/s: 75986.6 | physical tokens/s: 76667.3 | loss tokens/s: 55119.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.544624E-01 | lm-global-supervised-tokens: 1.216993E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.325005E-02 | lm-token-buffer-mib: 2.354894E+02 | global_load_balancing_loss: 9.990639E-01 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11144.677 | params rms: 0.060345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:35:10.798631 | finish at 2026-10-17 19:27:38 + [2026-10-17 10:56:08] iteration 2245/ 2384 | consumed samples: 71840 | elapsed time per iteration (ms): 220467.7 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 76098.1 | valid tokens/s: 76098.0 | physical tokens/s: 76497.4 | loss tokens/s: 57062.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.083470E-01 | lm-global-supervised-tokens: 1.258033E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.432294E-02 | lm-token-buffer-mib: 2.336987E+02 | global_load_balancing_loss: 9.998074E-01 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11144.651 | params rms: 0.060345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:30:45.009008 | finish at 2026-10-17 19:26:53 + [2026-10-17 10:59:48] iteration 2246/ 2384 | consumed samples: 71872 | elapsed time per iteration (ms): 219668.9 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 76374.9 | valid tokens/s: 76374.9 | physical tokens/s: 76947.5 | loss tokens/s: 55022.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.124114E-01 | lm-global-supervised-tokens: 1.208667E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.276779E-02 | lm-token-buffer-mib: 2.341283E+02 | global_load_balancing_loss: 9.991779E-01 | loss scale: 1.0 | grad norm: 0.231 | params norm: 11144.623 | params rms: 0.060345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:25:14.309584 | finish at 2026-10-17 19:25:02 + [2026-10-17 11:03:27] iteration 2247/ 2384 | consumed samples: 71904 | elapsed time per iteration (ms): 219915.2 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 76289.4 | valid tokens/s: 76289.4 | physical tokens/s: 76956.8 | loss tokens/s: 59158.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.469564E-01 | lm-global-supervised-tokens: 1.300981E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.681125E-02 | lm-token-buffer-mib: 2.337374E+02 | global_load_balancing_loss: 9.996982E-01 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11144.595 | params rms: 0.060344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:22:08.379884 | finish at 2026-10-17 19:25:36 + [2026-10-17 11:07:09] iteration 2248/ 2384 | consumed samples: 71936 | elapsed time per iteration (ms): 222037.7 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 75560.1 | valid tokens/s: 75560.1 | physical tokens/s: 76319.4 | loss tokens/s: 58996.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.437628E-01 | lm-global-supervised-tokens: 1.309934E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.457303E-02 | lm-token-buffer-mib: 2.348109E+02 | global_load_balancing_loss: 1.000916E+00 | loss scale: 1.0 | grad norm: 0.209 | params norm: 11144.565 | params rms: 0.060344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:23:17.133644 | finish at 2026-10-17 19:30:27 + [2026-10-17 11:10:57] iteration 2249/ 2384 | consumed samples: 71968 | elapsed time per iteration (ms): 227941.4 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 73603.2 | valid tokens/s: 73603.2 | physical tokens/s: 75480.7 | loss tokens/s: 56011.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.450254E-01 | lm-global-supervised-tokens: 1.276741E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.421192E-02 | lm-token-buffer-mib: 2.397916E+02 | global_load_balancing_loss: 1.000165E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11144.538 | params rms: 0.060344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:32:52.085305 | finish at 2026-10-17 19:43:49 + [2026-10-17 11:15:02] iteration 2250/ 2384 | consumed samples: 72000 | elapsed time per iteration (ms): 244967.3 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 68487.5 | valid tokens/s: 68487.5 | physical tokens/s: 68957.8 | loss tokens/s: 51839.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.011516E-01 | lm-global-supervised-tokens: 1.269907E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.127230E-02 | lm-token-buffer-mib: 2.340351E+02 | global_load_balancing_loss: 1.000745E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11144.511 | params rms: 0.060344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:07:05.624805 | finish at 2026-10-17 20:22:08 + [2026-10-17 11:18:46] iteration 2251/ 2384 | consumed samples: 72032 | elapsed time per iteration (ms): 223344.9 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75118.0 | valid tokens/s: 75118.0 | physical tokens/s: 75973.3 | loss tokens/s: 56318.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.820056E-01 | lm-global-supervised-tokens: 1.257849E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.511421E-02 | lm-token-buffer-mib: 2.339058E+02 | global_load_balancing_loss: 9.995587E-01 | loss scale: 1.0 | grad norm: 0.305 | params norm: 11144.483 | params rms: 0.060344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:15:04.869244 | finish at 2026-10-17 19:33:51 + [2026-10-17 11:22:29] iteration 2252/ 2384 | consumed samples: 72064 | elapsed time per iteration (ms): 223588.3 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75036.0 | valid tokens/s: 75036.0 | physical tokens/s: 75305.0 | loss tokens/s: 56610.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.558161E-01 | lm-global-supervised-tokens: 1.265742E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.194622E-02 | lm-token-buffer-mib: 2.326998E+02 | global_load_balancing_loss: 1.001279E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11144.456 | params rms: 0.060344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:11:53.661295 | finish at 2026-10-17 19:34:23 + [2026-10-17 11:26:11] iteration 2253/ 2384 | consumed samples: 72096 | elapsed time per iteration (ms): 221251.7 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 75828.6 | valid tokens/s: 75828.6 | physical tokens/s: 76812.1 | loss tokens/s: 57995.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.799558E-01 | lm-global-supervised-tokens: 1.283149E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.951302E-02 | lm-token-buffer-mib: 2.326909E+02 | global_load_balancing_loss: 1.000101E+00 | loss scale: 1.0 | grad norm: 0.344 | params norm: 11144.429 | params rms: 0.060343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:03:03.970629 | finish at 2026-10-17 19:29:15 + [2026-10-17 11:29:52] iteration 2254/ 2384 | consumed samples: 72128 | elapsed time per iteration (ms): 221554.7 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75724.9 | valid tokens/s: 75724.9 | physical tokens/s: 76719.8 | loss tokens/s: 59147.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.500040E-01 | lm-global-supervised-tokens: 1.310430E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.914534E-02 | lm-token-buffer-mib: 2.326736E+02 | global_load_balancing_loss: 1.000081E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11144.403 | params rms: 0.060343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:00:02.114551 | finish at 2026-10-17 19:29:54 + [2026-10-17 11:33:35] iteration 2255/ 2384 | consumed samples: 72160 | elapsed time per iteration (ms): 222391.6 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 75439.8 | valid tokens/s: 75439.8 | physical tokens/s: 76277.1 | loss tokens/s: 55289.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.149273E-01 | lm-global-supervised-tokens: 1.229580E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.477967E-02 | lm-token-buffer-mib: 2.357766E+02 | global_load_balancing_loss: 1.001442E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11144.376 | params rms: 0.060343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:58:08.511312 | finish at 2026-10-17 19:31:43 + [2026-10-17 11:37:18] iteration 2256/ 2384 | consumed samples: 72192 | elapsed time per iteration (ms): 223677.4 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.20% | FLOPs estimator: workload-aware | logical tokens/s: 75006.2 | valid tokens/s: 75006.2 | physical tokens/s: 76268.4 | loss tokens/s: 58928.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.811281E-01 | lm-global-supervised-tokens: 1.318092E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.863487E-02 | lm-token-buffer-mib: 2.335202E+02 | global_load_balancing_loss: 1.000221E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11144.349 | params rms: 0.060343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:57:10.713379 | finish at 2026-10-17 19:34:29 + [2026-10-17 11:41:02] iteration 2257/ 2384 | consumed samples: 72224 | elapsed time per iteration (ms): 223551.8 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75048.4 | valid tokens/s: 75048.4 | physical tokens/s: 75612.1 | loss tokens/s: 57454.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.846853E-01 | lm-global-supervised-tokens: 1.284397E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.059609E-02 | lm-token-buffer-mib: 2.341070E+02 | global_load_balancing_loss: 1.000856E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11144.321 | params rms: 0.060343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:53:11.081236 | finish at 2026-10-17 19:34:13 + [2026-10-17 11:44:45] iteration 2258/ 2384 | consumed samples: 72256 | elapsed time per iteration (ms): 223288.8 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 75136.7 | valid tokens/s: 75136.7 | physical tokens/s: 76000.7 | loss tokens/s: 58394.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.816934E-01 | lm-global-supervised-tokens: 1.303877E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.349487E-02 | lm-token-buffer-mib: 2.351922E+02 | global_load_balancing_loss: 9.996940E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11144.294 | params rms: 0.060343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:48:54.390603 | finish at 2026-10-17 19:33:39 + [2026-10-17 11:48:25] iteration 2259/ 2384 | consumed samples: 72288 | elapsed time per iteration (ms): 219679.9 | throughput per GPU (TFLOP/s/GPU): 17.6 | MFU 5.65% | FLOPs estimator: workload-aware | logical tokens/s: 76371.2 | valid tokens/s: 76371.2 | physical tokens/s: 77226.0 | loss tokens/s: 62187.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.593972E-01 | lm-global-supervised-tokens: 1.366132E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.892424E-02 | lm-token-buffer-mib: 2.346969E+02 | global_load_balancing_loss: 9.991267E-01 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11144.270 | params rms: 0.060343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:37:39.982336 | finish at 2026-10-17 19:26:05 + [2026-10-17 11:52:27] iteration 2260/ 2384 | consumed samples: 72320 | elapsed time per iteration (ms): 242035.5 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.32% | FLOPs estimator: workload-aware | logical tokens/s: 69317.0 | valid tokens/s: 69317.0 | physical tokens/s: 69686.1 | loss tokens/s: 53290.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.576082E-01 | lm-global-supervised-tokens: 1.289818E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.152134E-02 | lm-token-buffer-mib: 2.338270E+02 | global_load_balancing_loss: 9.999552E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11144.244 | params rms: 0.060342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:20:12.396152 | finish at 2026-10-17 20:12:39 + [2026-10-17 11:56:08] iteration 2261/ 2384 | consumed samples: 72352 | elapsed time per iteration (ms): 221690.8 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75678.3 | valid tokens/s: 75678.3 | physical tokens/s: 77137.8 | loss tokens/s: 58270.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.317357E-01 | lm-global-supervised-tokens: 1.291803E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.738289E-02 | lm-token-buffer-mib: 2.346518E+02 | global_load_balancing_loss: 9.998119E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11144.218 | params rms: 0.060342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:34:27.972148 | finish at 2026-10-17 19:30:36 + [2026-10-17 11:59:54] iteration 2262/ 2384 | consumed samples: 72384 | elapsed time per iteration (ms): 226027.1 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 74226.4 | valid tokens/s: 74226.3 | physical tokens/s: 75201.2 | loss tokens/s: 57901.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.674716E-01 | lm-global-supervised-tokens: 1.308728E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.868023E-02 | lm-token-buffer-mib: 2.369662E+02 | global_load_balancing_loss: 1.001412E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11144.190 | params rms: 0.060342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:39:35.307403 | finish at 2026-10-17 19:39:30 + [2026-10-17 12:03:40] iteration 2263/ 2384 | consumed samples: 72416 | elapsed time per iteration (ms): 225344.7 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.98% | FLOPs estimator: workload-aware | logical tokens/s: 74451.3 | valid tokens/s: 74451.3 | physical tokens/s: 74971.9 | loss tokens/s: 56783.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.338236E-01 | lm-global-supervised-tokens: 1.279575E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.193785E-02 | lm-token-buffer-mib: 2.330412E+02 | global_load_balancing_loss: 1.000914E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11144.164 | params rms: 0.060342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:34:26.713876 | finish at 2026-10-17 19:38:07 + [2026-10-17 12:07:21] iteration 2264/ 2384 | consumed samples: 72448 | elapsed time per iteration (ms): 220721.1 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 76010.7 | valid tokens/s: 76010.7 | physical tokens/s: 76674.1 | loss tokens/s: 53657.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.912462E-01 | lm-global-supervised-tokens: 1.184342E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.350939E-02 | lm-token-buffer-mib: 2.340166E+02 | global_load_balancing_loss: 1.001709E+00 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11144.139 | params rms: 0.060342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:21:26.535215 | finish at 2026-10-17 19:28:47 + [2026-10-17 12:11:05] iteration 2265/ 2384 | consumed samples: 72480 | elapsed time per iteration (ms): 224802.6 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 74630.8 | valid tokens/s: 74630.8 | physical tokens/s: 75712.7 | loss tokens/s: 59135.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.975692E-01 | lm-global-supervised-tokens: 1.329377E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.497095E-02 | lm-token-buffer-mib: 2.346071E+02 | global_load_balancing_loss: 1.000247E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11144.116 | params rms: 0.060342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:25:51.512340 | finish at 2026-10-17 19:36:57 + [2026-10-17 12:14:47] iteration 2266/ 2384 | consumed samples: 72512 | elapsed time per iteration (ms): 221264.5 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75824.1 | valid tokens/s: 75824.0 | physical tokens/s: 76949.7 | loss tokens/s: 50198.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.841061E-01 | lm-global-supervised-tokens: 1.110717E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.263110E-02 | lm-token-buffer-mib: 2.324573E+02 | global_load_balancing_loss: 9.990722E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11144.093 | params rms: 0.060342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:15:09.207022 | finish at 2026-10-17 19:29:56 + [2026-10-17 12:18:30] iteration 2267/ 2384 | consumed samples: 72544 | elapsed time per iteration (ms): 222945.7 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75252.4 | valid tokens/s: 75252.4 | physical tokens/s: 75779.5 | loss tokens/s: 56429.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.491651E-01 | lm-global-supervised-tokens: 1.258061E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.064877E-02 | lm-token-buffer-mib: 2.340102E+02 | global_load_balancing_loss: 9.986857E-01 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11144.069 | params rms: 0.060342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:14:44.650183 | finish at 2026-10-17 19:33:14 + [2026-10-17 12:22:15] iteration 2268/ 2384 | consumed samples: 72576 | elapsed time per iteration (ms): 225460.5 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 74413.0 | valid tokens/s: 74413.0 | physical tokens/s: 75485.3 | loss tokens/s: 53143.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.517222E-01 | lm-global-supervised-tokens: 1.198177E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.417457E-02 | lm-token-buffer-mib: 2.374818E+02 | global_load_balancing_loss: 1.000476E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11144.042 | params rms: 0.060341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:15:53.415539 | finish at 2026-10-17 19:38:08 + [2026-10-17 12:25:54] iteration 2269/ 2384 | consumed samples: 72608 | elapsed time per iteration (ms): 218944.8 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 76627.5 | valid tokens/s: 76627.5 | physical tokens/s: 76981.0 | loss tokens/s: 54123.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.495222E-01 | lm-global-supervised-tokens: 1.185008E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.183711E-02 | lm-token-buffer-mib: 2.331129E+02 | global_load_balancing_loss: 1.000528E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11144.017 | params rms: 0.060341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:59:38.657033 | finish at 2026-10-17 19:25:33 + [2026-10-17 12:29:56] iteration 2270/ 2384 | consumed samples: 72640 | elapsed time per iteration (ms): 242287.6 | throughput per GPU (TFLOP/s/GPU): 17.9 | MFU 5.74% | FLOPs estimator: workload-aware | logical tokens/s: 69245.0 | valid tokens/s: 69245.0 | physical tokens/s: 69780.2 | loss tokens/s: 53857.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.909510E-01 | lm-global-supervised-tokens: 1.304906E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.272904E-02 | lm-token-buffer-mib: 2.328821E+02 | global_load_balancing_loss: 1.000129E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11143.991 | params rms: 0.060341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:40:20.784502 | finish at 2026-10-17 20:10:17 + [2026-10-17 12:33:39] iteration 2271/ 2384 | consumed samples: 72672 | elapsed time per iteration (ms): 223020.5 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 75227.2 | valid tokens/s: 75227.2 | physical tokens/s: 76260.9 | loss tokens/s: 58613.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.119193E-01 | lm-global-supervised-tokens: 1.307196E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.022280E-02 | lm-token-buffer-mib: 2.357020E+02 | global_load_balancing_loss: 1.001539E+00 | loss scale: 1.0 | grad norm: 0.511 | params norm: 11143.965 | params rms: 0.060341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:00:01.311267 | finish at 2026-10-17 19:33:41 + [2026-10-17 12:37:20] iteration 2272/ 2384 | consumed samples: 72704 | elapsed time per iteration (ms): 221026.7 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.01% | FLOPs estimator: workload-aware | logical tokens/s: 75905.6 | valid tokens/s: 75905.6 | physical tokens/s: 76340.5 | loss tokens/s: 58592.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.139641E-01 | lm-global-supervised-tokens: 1.295039E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.242259E-02 | lm-token-buffer-mib: 2.336378E+02 | global_load_balancing_loss: 1.000760E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11143.940 | params rms: 0.060341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:52:34.993713 | finish at 2026-10-17 19:29:55 + [2026-10-17 12:41:02] iteration 2273/ 2384 | consumed samples: 72736 | elapsed time per iteration (ms): 221456.4 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 75758.5 | valid tokens/s: 75758.5 | physical tokens/s: 76748.4 | loss tokens/s: 58271.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.261549E-01 | lm-global-supervised-tokens: 1.290458E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.805943E-02 | lm-token-buffer-mib: 2.361598E+02 | global_load_balancing_loss: 9.996058E-01 | loss scale: 1.0 | grad norm: 0.452 | params norm: 11143.916 | params rms: 0.060341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:49:41.659147 | finish at 2026-10-17 19:30:43 + [2026-10-17 12:44:42] iteration 2274/ 2384 | consumed samples: 72768 | elapsed time per iteration (ms): 219834.1 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 76317.5 | valid tokens/s: 76317.5 | physical tokens/s: 76732.8 | loss tokens/s: 57895.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.490885E-01 | lm-global-supervised-tokens: 1.272745E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.846433E-02 | lm-token-buffer-mib: 2.325734E+02 | global_load_balancing_loss: 1.000163E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11143.888 | params rms: 0.060341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:43:01.746778 | finish at 2026-10-17 19:27:43 + [2026-10-17 12:48:23] iteration 2275/ 2384 | consumed samples: 72800 | elapsed time per iteration (ms): 221244.5 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 75831.1 | valid tokens/s: 75831.1 | physical tokens/s: 76639.9 | loss tokens/s: 57852.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.897244E-01 | lm-global-supervised-tokens: 1.279951E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.976185E-02 | lm-token-buffer-mib: 2.351001E+02 | global_load_balancing_loss: 9.996148E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11143.859 | params rms: 0.060340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:41:55.655039 | finish at 2026-10-17 19:30:18 + [2026-10-17 12:52:03] iteration 2276/ 2384 | consumed samples: 72832 | elapsed time per iteration (ms): 220179.0 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 76198.0 | valid tokens/s: 76198.0 | physical tokens/s: 76616.1 | loss tokens/s: 56819.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.169997E-01 | lm-global-supervised-tokens: 1.251040E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.711416E-02 | lm-token-buffer-mib: 2.323102E+02 | global_load_balancing_loss: 1.000439E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11143.827 | params rms: 0.060340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:36:19.333457 | finish at 2026-10-17 19:28:22 + [2026-10-17 12:55:44] iteration 2277/ 2384 | consumed samples: 72864 | elapsed time per iteration (ms): 221199.8 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 75846.3 | valid tokens/s: 75846.3 | physical tokens/s: 76499.4 | loss tokens/s: 57696.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.587345E-01 | lm-global-supervised-tokens: 1.276234E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.526522E-02 | lm-token-buffer-mib: 2.341816E+02 | global_load_balancing_loss: 1.000838E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11143.795 | params rms: 0.060340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:34:28.381892 | finish at 2026-10-17 19:30:13 + [2026-10-17 12:59:25] iteration 2278/ 2384 | consumed samples: 72896 | elapsed time per iteration (ms): 221204.9 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.04% | FLOPs estimator: workload-aware | logical tokens/s: 75844.4 | valid tokens/s: 75844.4 | physical tokens/s: 76326.7 | loss tokens/s: 56840.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.746645E-01 | lm-global-supervised-tokens: 1.257343E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.258635E-02 | lm-token-buffer-mib: 2.335439E+02 | global_load_balancing_loss: 1.000883E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11143.766 | params rms: 0.060340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:30:47.719984 | finish at 2026-10-17 19:30:13 + [2026-10-17 13:03:11] iteration 2279/ 2384 | consumed samples: 72928 | elapsed time per iteration (ms): 226049.2 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 74219.2 | valid tokens/s: 74219.2 | physical tokens/s: 75169.2 | loss tokens/s: 53947.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.584217E-01 | lm-global-supervised-tokens: 1.219468E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.712281E-02 | lm-token-buffer-mib: 2.343587E+02 | global_load_balancing_loss: 1.000250E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11143.733 | params rms: 0.060340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:35:35.169486 | finish at 2026-10-17 19:38:47 + [2026-10-17 13:07:17] iteration 2280/ 2384 | consumed samples: 72960 | elapsed time per iteration (ms): 245450.6 | throughput per GPU (TFLOP/s/GPU): 16.8 | MFU 5.39% | FLOPs estimator: workload-aware | logical tokens/s: 68352.7 | valid tokens/s: 68352.7 | physical tokens/s: 69710.4 | loss tokens/s: 53051.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.295183E-01 | lm-global-supervised-tokens: 1.302160E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.598435E-02 | lm-token-buffer-mib: 2.378257E+02 | global_load_balancing_loss: 1.000650E+00 | loss scale: 1.0 | grad norm: 0.305 | params norm: 11143.701 | params rms: 0.060340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:05:26.864151 | finish at 2026-10-17 20:12:44 +times across ranks (ms): + save-checkpoint: + rank 0: 24455.07 + rank 1: 24455.09 + rank 2: 24455.10 + rank 3: 24455.10 + rank 4: 24455.07 + rank 5: 24455.08 + rank 6: 24455.08 + rank 7: 24455.07 + rank 8: 24455.91 + rank 9: 24455.98 + rank 10: 24455.93 + rank 11: 24455.91 + rank 12: 24455.92 + rank 13: 24455.92 + rank 14: 24455.93 + rank 15: 24455.90 + rank 16: 24455.84 + rank 17: 24455.83 + rank 18: 24455.84 + rank 19: 24455.81 + rank 20: 24455.81 + rank 21: 24455.85 + rank 22: 24455.84 + rank 23: 24455.80 + rank 24: 24456.12 + rank 25: 24456.08 + rank 26: 24456.27 + rank 27: 24456.09 + rank 28: 24456.15 + rank 29: 24456.09 + rank 30: 24456.07 + rank 31: 24456.08 + rank 32: 24455.92 + rank 33: 24455.92 + rank 34: 24455.89 + rank 35: 24457.65 + rank 36: 24455.88 + rank 37: 24455.92 + rank 38: 24455.89 + rank 39: 24455.88 + rank 40: 24455.99 + rank 41: 24455.98 + rank 42: 24455.96 + rank 43: 24455.98 + rank 44: 24455.98 + rank 45: 24455.99 + rank 46: 24455.97 + rank 47: 24455.99 + rank 48: 24455.98 + rank 49: 24455.88 + rank 50: 24455.76 + rank 51: 24455.77 + rank 52: 24455.71 + rank 53: 24455.80 + rank 54: 24455.63 + rank 55: 24455.98 + rank 56: 24456.23 + rank 57: 24456.24 + rank 58: 24456.23 + rank 59: 24456.23 + rank 60: 24456.25 + rank 61: 24456.23 + rank 62: 24456.22 + rank 63: 24456.25 + [2026-10-17 13:11:21] iteration 2281/ 2384 | consumed samples: 72992 | elapsed time per iteration (ms): 219158.1 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.95% | FLOPs estimator: workload-aware | logical tokens/s: 76552.9 | valid tokens/s: 76552.9 | physical tokens/s: 77698.6 | loss tokens/s: 59736.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.364090E-01 | lm-global-supervised-tokens: 1.309167E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.113067E-01 | lm-token-buffer-mib: 2.334048E+02 | global_load_balancing_loss: 1.000092E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11143.672 | params rms: 0.060339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:16:13.286572 | finish at 2026-10-17 19:27:34 + [2026-10-17 13:15:09] iteration 2282/ 2384 | consumed samples: 73024 | elapsed time per iteration (ms): 228599.7 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 73391.1 | valid tokens/s: 73391.1 | physical tokens/s: 73896.6 | loss tokens/s: 52498.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.528279E-01 | lm-global-supervised-tokens: 1.200105E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.698593E-02 | lm-token-buffer-mib: 2.331409E+02 | global_load_balancing_loss: 1.000112E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11143.646 | params rms: 0.060339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:28:37.166503 | finish at 2026-10-17 19:43:46 + [2026-10-17 13:18:51] iteration 2283/ 2384 | consumed samples: 73056 | elapsed time per iteration (ms): 221727.8 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.99% | FLOPs estimator: workload-aware | logical tokens/s: 75665.7 | valid tokens/s: 75665.7 | physical tokens/s: 76177.0 | loss tokens/s: 62400.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.289020E-01 | lm-global-supervised-tokens: 1.383593E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.220312E-02 | lm-token-buffer-mib: 2.337020E+02 | global_load_balancing_loss: 9.992660E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11143.617 | params rms: 0.060339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:13:14.508728 | finish at 2026-10-17 19:32:05 + [2026-10-17 13:22:32] iteration 2284/ 2384 | consumed samples: 73088 | elapsed time per iteration (ms): 220705.5 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.25% | FLOPs estimator: workload-aware | logical tokens/s: 76016.2 | valid tokens/s: 76016.2 | physical tokens/s: 76413.9 | loss tokens/s: 58153.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.891984E-01 | lm-global-supervised-tokens: 1.283485E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.155112E-02 | lm-token-buffer-mib: 2.332994E+02 | global_load_balancing_loss: 1.001357E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11143.586 | params rms: 0.060339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:07:50.547628 | finish at 2026-10-17 19:30:22 + [2026-10-17 13:26:15] iteration 2285/ 2384 | consumed samples: 73120 | elapsed time per iteration (ms): 223005.0 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 75232.4 | valid tokens/s: 75232.4 | physical tokens/s: 75999.0 | loss tokens/s: 54609.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.354436E-01 | lm-global-supervised-tokens: 1.217812E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.253399E-02 | lm-token-buffer-mib: 2.325742E+02 | global_load_balancing_loss: 1.000973E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11143.553 | params rms: 0.060339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:07:57.493005 | finish at 2026-10-17 19:34:12 + [2026-10-17 13:30:00] iteration 2286/ 2384 | consumed samples: 73152 | elapsed time per iteration (ms): 225171.0 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.38% | FLOPs estimator: workload-aware | logical tokens/s: 74508.7 | valid tokens/s: 74508.7 | physical tokens/s: 75549.3 | loss tokens/s: 57031.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.113281E-01 | lm-global-supervised-tokens: 1.284180E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.260405E-02 | lm-token-buffer-mib: 2.359061E+02 | global_load_balancing_loss: 1.000389E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11143.523 | params rms: 0.060339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:07:46.762253 | finish at 2026-10-17 19:37:47 + [2026-10-17 13:33:42] iteration 2287/ 2384 | consumed samples: 73184 | elapsed time per iteration (ms): 222437.4 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.94% | FLOPs estimator: workload-aware | logical tokens/s: 75424.3 | valid tokens/s: 75424.3 | physical tokens/s: 75859.8 | loss tokens/s: 58429.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.149831E-01 | lm-global-supervised-tokens: 1.299691E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.555120E-02 | lm-token-buffer-mib: 2.323913E+02 | global_load_balancing_loss: 9.992817E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11143.493 | params rms: 0.060338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:59:36.423347 | finish at 2026-10-17 19:33:19 + [2026-10-17 13:37:27] iteration 2288/ 2384 | consumed samples: 73216 | elapsed time per iteration (ms): 224708.9 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.38% | FLOPs estimator: workload-aware | logical tokens/s: 74661.9 | valid tokens/s: 74661.9 | physical tokens/s: 75593.1 | loss tokens/s: 57803.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.612789E-01 | lm-global-supervised-tokens: 1.298902E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.528054E-02 | lm-token-buffer-mib: 2.331384E+02 | global_load_balancing_loss: 1.000718E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11143.462 | params rms: 0.060338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:59:32.052956 | finish at 2026-10-17 19:36:59 + [2026-10-17 13:41:07] iteration 2289/ 2384 | consumed samples: 73248 | elapsed time per iteration (ms): 220394.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 76123.6 | valid tokens/s: 76123.6 | physical tokens/s: 76526.2 | loss tokens/s: 53840.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.439551E-01 | lm-global-supervised-tokens: 1.186600E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.149053E-02 | lm-token-buffer-mib: 2.328395E+02 | global_load_balancing_loss: 9.997050E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11143.429 | params rms: 0.060338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:48:57.432179 | finish at 2026-10-17 19:30:05 + [2026-10-17 13:45:12] iteration 2290/ 2384 | consumed samples: 73280 | elapsed time per iteration (ms): 245183.0 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.48% | FLOPs estimator: workload-aware | logical tokens/s: 68427.1 | valid tokens/s: 68427.1 | physical tokens/s: 69137.1 | loss tokens/s: 50334.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.253222E-01 | lm-global-supervised-tokens: 1.234112E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.429187E-02 | lm-token-buffer-mib: 2.336276E+02 | global_load_balancing_loss: 1.000331E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11143.395 | params rms: 0.060338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:24:07.198669 | finish at 2026-10-17 20:09:20 + [2026-10-17 13:48:54] iteration 2291/ 2384 | consumed samples: 73312 | elapsed time per iteration (ms): 221562.5 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 75722.1 | valid tokens/s: 75722.1 | physical tokens/s: 76532.8 | loss tokens/s: 57487.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.228854E-01 | lm-global-supervised-tokens: 1.273715E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.664554E-02 | lm-token-buffer-mib: 2.355691E+02 | global_load_balancing_loss: 9.999793E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11143.361 | params rms: 0.060338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:43:25.315693 | finish at 2026-10-17 19:32:19 + [2026-10-17 13:52:34] iteration 2292/ 2384 | consumed samples: 73344 | elapsed time per iteration (ms): 220343.9 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.24% | FLOPs estimator: workload-aware | logical tokens/s: 76141.0 | valid tokens/s: 76141.0 | physical tokens/s: 76511.7 | loss tokens/s: 53016.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.174196E-01 | lm-global-supervised-tokens: 1.168191E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.144316E-02 | lm-token-buffer-mib: 2.332882E+02 | global_load_balancing_loss: 1.000248E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11143.332 | params rms: 0.060338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:37:51.640793 | finish at 2026-10-17 19:30:26 + [2026-10-17 13:56:19] iteration 2293/ 2384 | consumed samples: 73376 | elapsed time per iteration (ms): 224842.4 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 74617.6 | valid tokens/s: 74617.6 | physical tokens/s: 75367.4 | loss tokens/s: 53823.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.829046E-01 | lm-global-supervised-tokens: 1.210177E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.598059E-02 | lm-token-buffer-mib: 2.341285E+02 | global_load_balancing_loss: 1.002222E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11143.303 | params rms: 0.060337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:41:00.659123 | finish at 2026-10-17 19:37:20 + [2026-10-17 14:00:05] iteration 2294/ 2384 | consumed samples: 73408 | elapsed time per iteration (ms): 226071.8 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.05% | FLOPs estimator: workload-aware | logical tokens/s: 74211.7 | valid tokens/s: 74211.7 | physical tokens/s: 74633.4 | loss tokens/s: 56312.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.947216E-01 | lm-global-supervised-tokens: 1.273064E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.237818E-02 | lm-token-buffer-mib: 2.331212E+02 | global_load_balancing_loss: 1.001182E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11143.275 | params rms: 0.060337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:39:06.458824 | finish at 2026-10-17 19:39:12 + [2026-10-17 14:03:48] iteration 2295/ 2384 | consumed samples: 73440 | elapsed time per iteration (ms): 222781.6 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 75307.9 | valid tokens/s: 75307.9 | physical tokens/s: 75705.5 | loss tokens/s: 60359.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.577833E-01 | lm-global-supervised-tokens: 1.344699E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.129873E-02 | lm-token-buffer-mib: 2.327178E+02 | global_load_balancing_loss: 1.000868E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11143.247 | params rms: 0.060337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:30:27.564607 | finish at 2026-10-17 19:34:16 + [2026-10-17 14:07:28] iteration 2296/ 2384 | consumed samples: 73472 | elapsed time per iteration (ms): 220135.2 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 76213.1 | valid tokens/s: 76213.1 | physical tokens/s: 76554.6 | loss tokens/s: 53952.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.871600E-01 | lm-global-supervised-tokens: 1.187684E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.555639E-02 | lm-token-buffer-mib: 2.327280E+02 | global_load_balancing_loss: 9.993801E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11143.221 | params rms: 0.060337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:22:51.896553 | finish at 2026-10-17 19:30:20 + [2026-10-17 14:11:10] iteration 2297/ 2384 | consumed samples: 73504 | elapsed time per iteration (ms): 221756.2 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.06% | FLOPs estimator: workload-aware | logical tokens/s: 75656.0 | valid tokens/s: 75656.0 | physical tokens/s: 76608.5 | loss tokens/s: 53646.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.223454E-01 | lm-global-supervised-tokens: 1.189643E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.254446E-02 | lm-token-buffer-mib: 2.342397E+02 | global_load_balancing_loss: 9.994433E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11143.199 | params rms: 0.060337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:21:32.792808 | finish at 2026-10-17 19:32:43 + [2026-10-17 14:14:56] iteration 2298/ 2384 | consumed samples: 73536 | elapsed time per iteration (ms): 226355.1 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 74118.9 | valid tokens/s: 74118.9 | physical tokens/s: 75084.6 | loss tokens/s: 51888.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.180083E-01 | lm-global-supervised-tokens: 1.174529E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.522236E-02 | lm-token-buffer-mib: 2.368417E+02 | global_load_balancing_loss: 9.990797E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11143.175 | params rms: 0.060337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:24:26.534718 | finish at 2026-10-17 19:39:23 + [2026-10-17 14:18:40] iteration 2299/ 2384 | consumed samples: 73568 | elapsed time per iteration (ms): 223327.0 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 75123.8 | valid tokens/s: 75123.8 | physical tokens/s: 75889.4 | loss tokens/s: 56563.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.064775E-01 | lm-global-supervised-tokens: 1.263207E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.863480E-02 | lm-token-buffer-mib: 2.332564E+02 | global_load_balancing_loss: 9.996828E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11143.148 | params rms: 0.060337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:16:22.795215 | finish at 2026-10-17 19:35:02 + [2026-10-17 14:22:47] iteration 2300/ 2384 | consumed samples: 73600 | elapsed time per iteration (ms): 247540.3 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.15% | FLOPs estimator: workload-aware | logical tokens/s: 67775.7 | valid tokens/s: 67775.7 | physical tokens/s: 69468.1 | loss tokens/s: 51142.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.045185E-01 | lm-global-supervised-tokens: 1.265992E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.308685E-02 | lm-token-buffer-mib: 2.375474E+02 | global_load_balancing_loss: 1.001384E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11143.120 | params rms: 0.060336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:46:33.383529 | finish at 2026-10-17 20:09:21 + [2026-10-17 14:26:32] iteration 2301/ 2384 | consumed samples: 73632 | elapsed time per iteration (ms): 224360.9 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.78% | FLOPs estimator: workload-aware | logical tokens/s: 74777.7 | valid tokens/s: 74777.7 | physical tokens/s: 75756.8 | loss tokens/s: 54823.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.038017E-01 | lm-global-supervised-tokens: 1.230014E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.497056E-02 | lm-token-buffer-mib: 2.353905E+02 | global_load_balancing_loss: 1.000455E+00 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11143.093 | params rms: 0.060336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:10:21.957979 | finish at 2026-10-17 19:36:54 + [2026-10-17 14:30:12] iteration 2302/ 2384 | consumed samples: 73664 | elapsed time per iteration (ms): 220219.0 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.53% | FLOPs estimator: workload-aware | logical tokens/s: 76184.1 | valid tokens/s: 76184.1 | physical tokens/s: 76604.8 | loss tokens/s: 59625.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.226683E-01 | lm-global-supervised-tokens: 1.313061E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.561822E-02 | lm-token-buffer-mib: 2.326640E+02 | global_load_balancing_loss: 1.002509E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11143.067 | params rms: 0.060336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:00:57.960100 | finish at 2026-10-17 19:31:10 + [2026-10-17 14:33:56] iteration 2303/ 2384 | consumed samples: 73696 | elapsed time per iteration (ms): 224515.3 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.76% | FLOPs estimator: workload-aware | logical tokens/s: 74726.3 | valid tokens/s: 74726.3 | physical tokens/s: 75674.8 | loss tokens/s: 56920.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.060522E-01 | lm-global-supervised-tokens: 1.277960E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.134936E-02 | lm-token-buffer-mib: 2.346332E+02 | global_load_balancing_loss: 1.000821E+00 | loss scale: 1.0 | grad norm: 0.269 | params norm: 11143.039 | params rms: 0.060336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:03:05.742895 | finish at 2026-10-17 19:37:02 + [2026-10-17 14:37:38] iteration 2304/ 2384 | consumed samples: 73728 | elapsed time per iteration (ms): 221528.1 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 75733.9 | valid tokens/s: 75733.9 | physical tokens/s: 76449.3 | loss tokens/s: 58403.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.139326E-01 | lm-global-supervised-tokens: 1.293804E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.527186E-02 | lm-token-buffer-mib: 2.345786E+02 | global_load_balancing_loss: 1.001765E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11143.011 | params rms: 0.060336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:55:22.251587 | finish at 2026-10-17 19:33:00 + [2026-10-17 14:41:19] iteration 2305/ 2384 | consumed samples: 73760 | elapsed time per iteration (ms): 220915.4 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 75943.9 | valid tokens/s: 75943.9 | physical tokens/s: 76692.7 | loss tokens/s: 57551.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.792441E-01 | lm-global-supervised-tokens: 1.271408E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.459019E-02 | lm-token-buffer-mib: 2.340327E+02 | global_load_balancing_loss: 1.000462E+00 | loss scale: 1.0 | grad norm: 0.235 | params norm: 11142.984 | params rms: 0.060336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:50:52.320068 | finish at 2026-10-17 19:32:11 + [2026-10-17 14:45:06] iteration 2306/ 2384 | consumed samples: 73792 | elapsed time per iteration (ms): 227255.5 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.32% | FLOPs estimator: workload-aware | logical tokens/s: 73825.3 | valid tokens/s: 73825.3 | physical tokens/s: 74831.6 | loss tokens/s: 54876.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.866845E-01 | lm-global-supervised-tokens: 1.247103E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.743218E-02 | lm-token-buffer-mib: 2.362440E+02 | global_load_balancing_loss: 1.000930E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11142.957 | params rms: 0.060335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:55:25.932279 | finish at 2026-10-17 19:40:32 + [2026-10-17 14:48:52] iteration 2307/ 2384 | consumed samples: 73824 | elapsed time per iteration (ms): 226389.5 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 74107.6 | valid tokens/s: 74107.6 | physical tokens/s: 74608.7 | loss tokens/s: 56267.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.210357E-01 | lm-global-supervised-tokens: 1.273847E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.131558E-02 | lm-token-buffer-mib: 2.337512E+02 | global_load_balancing_loss: 1.000165E+00 | loss scale: 1.0 | grad norm: 0.259 | params norm: 11142.931 | params rms: 0.060335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:50:31.992300 | finish at 2026-10-17 19:39:24 + [2026-10-17 14:52:41] iteration 2308/ 2384 | consumed samples: 73856 | elapsed time per iteration (ms): 229083.2 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.05% | FLOPs estimator: workload-aware | logical tokens/s: 73236.1 | valid tokens/s: 73236.1 | physical tokens/s: 74648.3 | loss tokens/s: 55836.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.880574E-01 | lm-global-supervised-tokens: 1.279118E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.593105E-02 | lm-token-buffer-mib: 2.370252E+02 | global_load_balancing_loss: 9.988361E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11142.905 | params rms: 0.060335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:50:10.323778 | finish at 2026-10-17 19:42:52 + [2026-10-17 14:56:24] iteration 2309/ 2384 | consumed samples: 73888 | elapsed time per iteration (ms): 222803.7 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 75300.3 | valid tokens/s: 75300.3 | physical tokens/s: 76066.5 | loss tokens/s: 53423.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.343751E-01 | lm-global-supervised-tokens: 1.190292E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.933865E-02 | lm-token-buffer-mib: 2.323668E+02 | global_load_balancing_loss: 9.994454E-01 | loss scale: 1.0 | grad norm: 0.238 | params norm: 11142.877 | params rms: 0.060335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:38:30.274386 | finish at 2026-10-17 19:34:55 + [2026-10-17 15:00:28] iteration 2310/ 2384 | consumed samples: 73920 | elapsed time per iteration (ms): 243272.4 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 68964.6 | valid tokens/s: 68964.6 | physical tokens/s: 69807.4 | loss tokens/s: 50433.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.468100E-01 | lm-global-supervised-tokens: 1.226904E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.376603E-02 | lm-token-buffer-mib: 2.339773E+02 | global_load_balancing_loss: 1.000479E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11142.846 | params rms: 0.060335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:00:02.155264 | finish at 2026-10-17 20:00:30 + [2026-10-17 15:04:07] iteration 2311/ 2384 | consumed samples: 73952 | elapsed time per iteration (ms): 219822.0 | throughput per GPU (TFLOP/s/GPU): 25.4 | MFU 8.14% | FLOPs estimator: workload-aware | logical tokens/s: 76321.7 | valid tokens/s: 76321.7 | physical tokens/s: 77097.1 | loss tokens/s: 56946.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.398970E-01 | lm-global-supervised-tokens: 1.251814E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.563748E-02 | lm-token-buffer-mib: 2.333123E+02 | global_load_balancing_loss: 1.000817E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11142.817 | params rms: 0.060335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:27:27.004505 | finish at 2026-10-17 19:31:34 + [2026-10-17 15:07:51] iteration 2312/ 2384 | consumed samples: 73984 | elapsed time per iteration (ms): 223593.4 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 75034.3 | valid tokens/s: 75034.3 | physical tokens/s: 75507.9 | loss tokens/s: 56914.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.744389E-01 | lm-global-supervised-tokens: 1.272562E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.293229E-02 | lm-token-buffer-mib: 2.338450E+02 | global_load_balancing_loss: 9.997063E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11142.791 | params rms: 0.060335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:28:18.727581 | finish at 2026-10-17 19:36:10 + [2026-10-17 15:11:32] iteration 2313/ 2384 | consumed samples: 74016 | elapsed time per iteration (ms): 220914.3 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.45% | FLOPs estimator: workload-aware | logical tokens/s: 75944.4 | valid tokens/s: 75944.4 | physical tokens/s: 76462.2 | loss tokens/s: 55675.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.420967E-01 | lm-global-supervised-tokens: 1.229947E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.536924E-02 | lm-token-buffer-mib: 2.339887E+02 | global_load_balancing_loss: 1.000504E+00 | loss scale: 1.0 | grad norm: 0.183 | params norm: 11142.765 | params rms: 0.060334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:21:24.911793 | finish at 2026-10-17 19:32:57 + [2026-10-17 15:15:12] iteration 2314/ 2384 | consumed samples: 74048 | elapsed time per iteration (ms): 220152.9 | throughput per GPU (TFLOP/s/GPU): 17.6 | MFU 5.63% | FLOPs estimator: workload-aware | logical tokens/s: 76207.1 | valid tokens/s: 76207.1 | physical tokens/s: 77120.5 | loss tokens/s: 56142.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.945115E-01 | lm-global-supervised-tokens: 1.236001E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.561167E-02 | lm-token-buffer-mib: 2.343414E+02 | global_load_balancing_loss: 1.000389E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11142.740 | params rms: 0.060334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:16:50.705051 | finish at 2026-10-17 19:32:03 + [2026-10-17 15:18:54] iteration 2315/ 2384 | consumed samples: 74080 | elapsed time per iteration (ms): 221695.2 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 75676.9 | valid tokens/s: 75676.9 | physical tokens/s: 76038.1 | loss tokens/s: 56184.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.243334E-01 | lm-global-supervised-tokens: 1.245584E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.272478E-02 | lm-token-buffer-mib: 2.328731E+02 | global_load_balancing_loss: 1.001852E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11142.719 | params rms: 0.060334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:14:56.968107 | finish at 2026-10-17 19:33:51 + [2026-10-17 15:22:34] iteration 2316/ 2384 | consumed samples: 74112 | elapsed time per iteration (ms): 220679.8 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 76025.1 | valid tokens/s: 76025.1 | physical tokens/s: 76927.4 | loss tokens/s: 57459.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.761549E-01 | lm-global-supervised-tokens: 1.268020E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.354721E-02 | lm-token-buffer-mib: 2.334173E+02 | global_load_balancing_loss: 9.994774E-01 | loss scale: 1.0 | grad norm: 0.433 | params norm: 11142.699 | params rms: 0.060334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:10:06.225689 | finish at 2026-10-17 19:32:41 + [2026-10-17 15:26:19] iteration 2317/ 2384 | consumed samples: 74144 | elapsed time per iteration (ms): 224222.7 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.24% | FLOPs estimator: workload-aware | logical tokens/s: 74823.7 | valid tokens/s: 74823.7 | physical tokens/s: 75296.3 | loss tokens/s: 57827.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.397504E-01 | lm-global-supervised-tokens: 1.296621E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.010002E-02 | lm-token-buffer-mib: 2.335825E+02 | global_load_balancing_loss: 9.999289E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11142.678 | params rms: 0.060334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:10:22.923671 | finish at 2026-10-17 19:36:42 + [2026-10-17 15:29:59] iteration 2318/ 2384 | consumed samples: 74176 | elapsed time per iteration (ms): 220382.6 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 76127.6 | valid tokens/s: 76127.6 | physical tokens/s: 76590.3 | loss tokens/s: 58933.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.694113E-01 | lm-global-supervised-tokens: 1.298783E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.136398E-02 | lm-token-buffer-mib: 2.335772E+02 | global_load_balancing_loss: 1.000900E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11142.656 | params rms: 0.060334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:02:25.251148 | finish at 2026-10-17 19:32:24 + [2026-10-17 15:33:41] iteration 2319/ 2384 | consumed samples: 74208 | elapsed time per iteration (ms): 221634.1 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 75697.8 | valid tokens/s: 75697.8 | physical tokens/s: 76292.4 | loss tokens/s: 56863.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.824160E-01 | lm-global-supervised-tokens: 1.260288E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.532175E-02 | lm-token-buffer-mib: 2.335546E+02 | global_load_balancing_loss: 1.000881E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11142.630 | params rms: 0.060334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:00:06.216916 | finish at 2026-10-17 19:33:47 + [2026-10-17 15:37:49] iteration 2320/ 2384 | consumed samples: 74240 | elapsed time per iteration (ms): 248417.0 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 67536.4 | valid tokens/s: 67536.4 | physical tokens/s: 67969.3 | loss tokens/s: 51848.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.987627E-01 | lm-global-supervised-tokens: 1.288000E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.198783E-02 | lm-token-buffer-mib: 2.329819E+02 | global_load_balancing_loss: 9.997357E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11142.600 | params rms: 0.060334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:24:58.690857 | finish at 2026-10-17 20:02:48 +times across ranks (ms): + save-checkpoint: + rank 0: 36303.55 + rank 1: 36303.59 + rank 2: 36303.59 + rank 3: 36303.58 + rank 4: 36303.55 + rank 5: 36303.57 + rank 6: 36303.58 + rank 7: 36303.56 + rank 8: 36302.62 + rank 9: 36302.69 + rank 10: 36302.79 + rank 11: 36302.66 + rank 12: 36302.72 + rank 13: 36302.59 + rank 14: 36302.74 + rank 15: 36302.66 + rank 16: 36303.46 + rank 17: 36303.50 + rank 18: 36303.46 + rank 19: 36303.46 + rank 20: 36303.44 + rank 21: 36303.47 + rank 22: 36303.46 + rank 23: 36303.43 + rank 24: 36303.64 + rank 25: 36303.61 + rank 26: 36303.63 + rank 27: 36303.60 + rank 28: 36303.59 + rank 29: 36304.47 + rank 30: 36303.61 + rank 31: 36303.61 + rank 32: 36303.33 + rank 33: 36303.25 + rank 34: 36303.29 + rank 35: 36303.31 + rank 36: 36303.22 + rank 37: 36303.28 + rank 38: 36303.24 + rank 39: 36303.24 + rank 40: 36301.75 + rank 41: 36301.75 + rank 42: 36301.74 + rank 43: 36301.79 + rank 44: 36301.70 + rank 45: 36301.71 + rank 46: 36301.58 + rank 47: 36301.72 + rank 48: 36303.36 + rank 49: 36303.29 + rank 50: 36303.39 + rank 51: 36303.38 + rank 52: 36303.33 + rank 53: 36303.44 + rank 54: 36303.32 + rank 55: 36303.30 + rank 56: 36303.53 + rank 57: 36303.58 + rank 58: 36303.40 + rank 59: 36303.57 + rank 60: 36303.58 + rank 61: 36303.54 + rank 62: 36303.55 + rank 63: 36303.53 + [2026-10-17 15:42:20] iteration 2321/ 2384 | consumed samples: 74272 | elapsed time per iteration (ms): 229958.6 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 72957.4 | valid tokens/s: 72957.4 | physical tokens/s: 73293.5 | loss tokens/s: 53280.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.565833E-01 | lm-global-supervised-tokens: 1.225232E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.579025E-02 | lm-token-buffer-mib: 2.327137E+02 | global_load_balancing_loss: 1.000905E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11142.573 | params rms: 0.060333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:01:27.389460 | finish at 2026-10-17 19:43:48 + [2026-10-17 15:46:10] iteration 2322/ 2384 | consumed samples: 74304 | elapsed time per iteration (ms): 229416.6 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 73129.9 | valid tokens/s: 73129.9 | physical tokens/s: 74593.9 | loss tokens/s: 54696.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.530672E-01 | lm-global-supervised-tokens: 1.254838E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.995479E-02 | lm-token-buffer-mib: 2.353427E+02 | global_load_balancing_loss: 1.000762E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11142.551 | params rms: 0.060333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:57:03.831180 | finish at 2026-10-17 19:43:14 + [2026-10-17 15:49:51] iteration 2323/ 2384 | consumed samples: 74336 | elapsed time per iteration (ms): 221011.8 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 75910.8 | valid tokens/s: 75910.8 | physical tokens/s: 76786.9 | loss tokens/s: 59353.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.614026E-01 | lm-global-supervised-tokens: 1.311792E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.104485E-02 | lm-token-buffer-mib: 2.341557E+02 | global_load_balancing_loss: 1.000097E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11142.527 | params rms: 0.060333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:44:41.719498 | finish at 2026-10-17 19:34:32 + [2026-10-17 15:53:30] iteration 2324/ 2384 | consumed samples: 74368 | elapsed time per iteration (ms): 218765.4 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 76690.4 | valid tokens/s: 76690.4 | physical tokens/s: 77305.1 | loss tokens/s: 56245.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.386979E-01 | lm-global-supervised-tokens: 1.230466E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.188881E-02 | lm-token-buffer-mib: 2.334958E+02 | global_load_balancing_loss: 9.994435E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11142.500 | params rms: 0.060333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:38:45.921221 | finish at 2026-10-17 19:32:15 + [2026-10-17 15:57:12] iteration 2325/ 2384 | consumed samples: 74400 | elapsed time per iteration (ms): 222177.3 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 75512.6 | valid tokens/s: 75512.6 | physical tokens/s: 75913.7 | loss tokens/s: 56758.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.969793E-01 | lm-global-supervised-tokens: 1.261041E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.608286E-02 | lm-token-buffer-mib: 2.330623E+02 | global_load_balancing_loss: 1.000032E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11142.471 | params rms: 0.060333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:38:28.459025 | finish at 2026-10-17 19:35:40 + [2026-10-17 16:00:51] iteration 2326/ 2384 | consumed samples: 74432 | elapsed time per iteration (ms): 219097.9 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 76574.0 | valid tokens/s: 76574.0 | physical tokens/s: 76898.0 | loss tokens/s: 55563.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.678597E-01 | lm-global-supervised-tokens: 1.217386E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.225894E-02 | lm-token-buffer-mib: 2.328466E+02 | global_load_balancing_loss: 1.000828E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11142.441 | params rms: 0.060333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:31:47.678762 | finish at 2026-10-17 19:32:38 + [2026-10-17 16:04:30] iteration 2327/ 2384 | consumed samples: 74464 | elapsed time per iteration (ms): 219356.4 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 76483.8 | valid tokens/s: 76483.8 | physical tokens/s: 77219.4 | loss tokens/s: 59733.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.659889E-01 | lm-global-supervised-tokens: 1.310292E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.370992E-02 | lm-token-buffer-mib: 2.345454E+02 | global_load_balancing_loss: 1.000194E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11142.410 | params rms: 0.060333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:28:23.313442 | finish at 2026-10-17 19:32:53 + [2026-10-17 16:08:09] iteration 2328/ 2384 | consumed samples: 74496 | elapsed time per iteration (ms): 219319.7 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.02% | FLOPs estimator: workload-aware | logical tokens/s: 76496.5 | valid tokens/s: 76496.5 | physical tokens/s: 77227.0 | loss tokens/s: 57607.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.356563E-01 | lm-global-supervised-tokens: 1.263438E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.509279E-02 | lm-token-buffer-mib: 2.336019E+02 | global_load_balancing_loss: 1.000604E+00 | loss scale: 1.0 | grad norm: 0.207 | params norm: 11142.379 | params rms: 0.060332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:24:41.904362 | finish at 2026-10-17 19:32:51 + [2026-10-17 16:11:51] iteration 2329/ 2384 | consumed samples: 74528 | elapsed time per iteration (ms): 221477.6 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 75751.2 | valid tokens/s: 75751.2 | physical tokens/s: 76335.3 | loss tokens/s: 60117.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.960168E-01 | lm-global-supervised-tokens: 1.331460E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.609985E-02 | lm-token-buffer-mib: 2.337760E+02 | global_load_balancing_loss: 1.000257E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11142.350 | params rms: 0.060332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:23:01.269540 | finish at 2026-10-17 19:34:52 + [2026-10-17 16:15:57] iteration 2330/ 2384 | consumed samples: 74560 | elapsed time per iteration (ms): 245585.8 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 68314.8 | valid tokens/s: 68314.8 | physical tokens/s: 68774.0 | loss tokens/s: 52462.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.856417E-01 | lm-global-supervised-tokens: 1.288399E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.156135E-02 | lm-token-buffer-mib: 2.341425E+02 | global_load_balancing_loss: 1.000507E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11142.319 | params rms: 0.060332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:41:01.632720 | finish at 2026-10-17 19:56:58 + [2026-10-17 16:19:36] iteration 2331/ 2384 | consumed samples: 74592 | elapsed time per iteration (ms): 219649.5 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 76381.6 | valid tokens/s: 76381.6 | physical tokens/s: 77217.4 | loss tokens/s: 59750.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.341751E-01 | lm-global-supervised-tokens: 1.312418E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.954121E-02 | lm-token-buffer-mib: 2.337741E+02 | global_load_balancing_loss: 1.000044E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11142.286 | params rms: 0.060332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:14:01.421068 | finish at 2026-10-17 19:33:38 + [2026-10-17 16:23:21] iteration 2332/ 2384 | consumed samples: 74624 | elapsed time per iteration (ms): 224805.8 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 74629.7 | valid tokens/s: 74629.7 | physical tokens/s: 75515.8 | loss tokens/s: 59408.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.581668E-01 | lm-global-supervised-tokens: 1.335530E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.531184E-02 | lm-token-buffer-mib: 2.344766E+02 | global_load_balancing_loss: 9.991755E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11142.254 | params rms: 0.060332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:14:49.902429 | finish at 2026-10-17 19:38:11 + [2026-10-17 16:27:02] iteration 2333/ 2384 | consumed samples: 74656 | elapsed time per iteration (ms): 220792.8 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.82% | FLOPs estimator: workload-aware | logical tokens/s: 75986.1 | valid tokens/s: 75986.1 | physical tokens/s: 76468.9 | loss tokens/s: 59069.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.034118E-01 | lm-global-supervised-tokens: 1.304219E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.515930E-02 | lm-token-buffer-mib: 2.328151E+02 | global_load_balancing_loss: 1.000955E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11142.220 | params rms: 0.060331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:07:40.430755 | finish at 2026-10-17 19:34:42 + [2026-10-17 16:30:49] iteration 2334/ 2384 | consumed samples: 74688 | elapsed time per iteration (ms): 227078.3 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.30% | FLOPs estimator: workload-aware | logical tokens/s: 73882.9 | valid tokens/s: 73882.9 | physical tokens/s: 74494.5 | loss tokens/s: 51951.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.306235E-01 | lm-global-supervised-tokens: 1.179696E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.837207E-02 | lm-token-buffer-mib: 2.345834E+02 | global_load_balancing_loss: 1.000315E+00 | loss scale: 1.0 | grad norm: 0.226 | params norm: 11142.187 | params rms: 0.060331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:09:13.915489 | finish at 2026-10-17 19:40:03 + [2026-10-17 16:34:32] iteration 2335/ 2384 | consumed samples: 74720 | elapsed time per iteration (ms): 223108.0 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.82% | FLOPs estimator: workload-aware | logical tokens/s: 75197.6 | valid tokens/s: 75197.6 | physical tokens/s: 75558.0 | loss tokens/s: 54148.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.969828E-01 | lm-global-supervised-tokens: 1.208089E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.226118E-02 | lm-token-buffer-mib: 2.331871E+02 | global_load_balancing_loss: 9.999257E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11142.150 | params rms: 0.060331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:02:12.293953 | finish at 2026-10-17 19:36:44 + [2026-10-17 16:38:15] iteration 2336/ 2384 | consumed samples: 74752 | elapsed time per iteration (ms): 223374.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 75108.0 | valid tokens/s: 75108.0 | physical tokens/s: 75705.2 | loss tokens/s: 57734.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.837619E-01 | lm-global-supervised-tokens: 1.289633E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.215733E-02 | lm-token-buffer-mib: 2.352695E+02 | global_load_balancing_loss: 9.994578E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11142.115 | params rms: 0.060331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:58:41.962681 | finish at 2026-10-17 19:36:57 + [2026-10-17 16:41:59] iteration 2337/ 2384 | consumed samples: 74784 | elapsed time per iteration (ms): 223408.2 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 75096.4 | valid tokens/s: 75096.4 | physical tokens/s: 75861.3 | loss tokens/s: 57395.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.888664E-01 | lm-global-supervised-tokens: 1.282271E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.806698E-02 | lm-token-buffer-mib: 2.354241E+02 | global_load_balancing_loss: 9.999607E-01 | loss scale: 1.0 | grad norm: 0.361 | params norm: 11142.081 | params rms: 0.060331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:55:00.187743 | finish at 2026-10-17 19:36:59 + [2026-10-17 16:45:43] iteration 2338/ 2384 | consumed samples: 74816 | elapsed time per iteration (ms): 223917.7 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 74925.8 | valid tokens/s: 74925.8 | physical tokens/s: 75420.3 | loss tokens/s: 57730.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.627765E-01 | lm-global-supervised-tokens: 1.292690E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.452816E-02 | lm-token-buffer-mib: 2.343448E+02 | global_load_balancing_loss: 1.001360E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11142.048 | params rms: 0.060331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:51:40.213248 | finish at 2026-10-17 19:37:23 + [2026-10-17 16:49:25] iteration 2339/ 2384 | consumed samples: 74848 | elapsed time per iteration (ms): 222591.3 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 75372.2 | valid tokens/s: 75372.2 | physical tokens/s: 75849.6 | loss tokens/s: 54607.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.999203E-01 | lm-global-supervised-tokens: 1.215518E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.255970E-02 | lm-token-buffer-mib: 2.342992E+02 | global_load_balancing_loss: 1.001550E+00 | loss scale: 1.0 | grad norm: 0.199 | params norm: 11142.013 | params rms: 0.060330 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:46:56.606601 | finish at 2026-10-17 19:36:22 + [2026-10-17 16:53:27] iteration 2340/ 2384 | consumed samples: 74880 | elapsed time per iteration (ms): 242071.6 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 69306.7 | valid tokens/s: 69306.7 | physical tokens/s: 69848.3 | loss tokens/s: 50554.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.586672E-01 | lm-global-supervised-tokens: 1.223779E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.947612E-02 | lm-token-buffer-mib: 2.333310E+02 | global_load_balancing_loss: 1.000599E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11141.978 | params rms: 0.060330 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:57:31.151699 | finish at 2026-10-17 19:50:59 + [2026-10-17 16:57:09] iteration 2341/ 2384 | consumed samples: 74912 | elapsed time per iteration (ms): 221548.4 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 75727.0 | valid tokens/s: 75727.0 | physical tokens/s: 76855.4 | loss tokens/s: 59222.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.167865E-01 | lm-global-supervised-tokens: 1.312058E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.650802E-02 | lm-token-buffer-mib: 2.361855E+02 | global_load_balancing_loss: 1.000565E+00 | loss scale: 1.0 | grad norm: 0.473 | params norm: 11141.942 | params rms: 0.060330 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:38:46.583278 | finish at 2026-10-17 19:35:55 + [2026-10-17 17:00:54] iteration 2342/ 2384 | consumed samples: 74944 | elapsed time per iteration (ms): 225490.2 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 74403.2 | valid tokens/s: 74403.2 | physical tokens/s: 75749.5 | loss tokens/s: 55230.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.144471E-01 | lm-global-supervised-tokens: 1.245383E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.656024E-02 | lm-token-buffer-mib: 2.343094E+02 | global_load_balancing_loss: 1.000049E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11141.908 | params rms: 0.060330 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:37:50.587040 | finish at 2026-10-17 19:38:45 + [2026-10-17 17:04:36] iteration 2343/ 2384 | consumed samples: 74976 | elapsed time per iteration (ms): 221375.3 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75786.2 | valid tokens/s: 75786.2 | physical tokens/s: 76175.7 | loss tokens/s: 59283.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.388635E-01 | lm-global-supervised-tokens: 1.312394E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.272259E-02 | lm-token-buffer-mib: 2.334174E+02 | global_load_balancing_loss: 9.992261E-01 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11141.875 | params rms: 0.060330 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:31:16.385499 | finish at 2026-10-17 19:35:52 + [2026-10-17 17:08:17] iteration 2344/ 2384 | consumed samples: 75008 | elapsed time per iteration (ms): 221481.6 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 75749.9 | valid tokens/s: 75749.9 | physical tokens/s: 76234.2 | loss tokens/s: 61025.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.651194E-01 | lm-global-supervised-tokens: 1.351600E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.207645E-02 | lm-token-buffer-mib: 2.333788E+02 | global_load_balancing_loss: 9.984977E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11141.844 | params rms: 0.060329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:27:39.265194 | finish at 2026-10-17 19:35:57 + [2026-10-17 17:11:59] iteration 2345/ 2384 | consumed samples: 75040 | elapsed time per iteration (ms): 221934.5 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.83% | FLOPs estimator: workload-aware | logical tokens/s: 75595.2 | valid tokens/s: 75595.2 | physical tokens/s: 76018.7 | loss tokens/s: 53326.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.831084E-01 | lm-global-supervised-tokens: 1.183504E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.653419E-02 | lm-token-buffer-mib: 2.341877E+02 | global_load_balancing_loss: 1.000462E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11141.817 | params rms: 0.060329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:24:15.445295 | finish at 2026-10-17 19:36:15 + [2026-10-17 17:15:46] iteration 2346/ 2384 | consumed samples: 75072 | elapsed time per iteration (ms): 227001.3 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.79% | FLOPs estimator: workload-aware | logical tokens/s: 73907.7 | valid tokens/s: 73907.7 | physical tokens/s: 74597.1 | loss tokens/s: 56848.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.957907E-01 | lm-global-supervised-tokens: 1.290466E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.462607E-02 | lm-token-buffer-mib: 2.343245E+02 | global_load_balancing_loss: 1.001882E+00 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11141.789 | params rms: 0.060329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:23:46.048461 | finish at 2026-10-17 19:39:32 + [2026-10-17 17:19:28] iteration 2347/ 2384 | consumed samples: 75104 | elapsed time per iteration (ms): 221818.9 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75634.6 | valid tokens/s: 75634.6 | physical tokens/s: 76210.6 | loss tokens/s: 56872.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.993708E-01 | lm-global-supervised-tokens: 1.261539E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.734344E-02 | lm-token-buffer-mib: 2.333898E+02 | global_load_balancing_loss: 1.001345E+00 | loss scale: 1.0 | grad norm: 0.212 | params norm: 11141.763 | params rms: 0.060329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:16:47.299542 | finish at 2026-10-17 19:36:15 + [2026-10-17 17:23:15] iteration 2348/ 2384 | consumed samples: 75136 | elapsed time per iteration (ms): 226644.5 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.68% | FLOPs estimator: workload-aware | logical tokens/s: 74024.3 | valid tokens/s: 74024.3 | physical tokens/s: 74776.9 | loss tokens/s: 58299.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.663527E-01 | lm-global-supervised-tokens: 1.321336E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.794252E-02 | lm-token-buffer-mib: 2.350932E+02 | global_load_balancing_loss: 1.001446E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11141.739 | params rms: 0.060329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:15:59.203219 | finish at 2026-10-17 19:39:14 + [2026-10-17 17:26:56] iteration 2349/ 2384 | consumed samples: 75168 | elapsed time per iteration (ms): 221012.7 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 75910.6 | valid tokens/s: 75910.6 | physical tokens/s: 76674.8 | loss tokens/s: 54251.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.748840E-01 | lm-global-supervised-tokens: 1.199023E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.141776E-02 | lm-token-buffer-mib: 2.338506E+02 | global_load_balancing_loss: 1.000670E+00 | loss scale: 1.0 | grad norm: 0.180 | params norm: 11141.714 | params rms: 0.060329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:08:55.445604 | finish at 2026-10-17 19:35:51 + [2026-10-17 17:31:03] iteration 2350/ 2384 | consumed samples: 75200 | elapsed time per iteration (ms): 247514.5 | throughput per GPU (TFLOP/s/GPU): 17.1 | MFU 5.49% | FLOPs estimator: workload-aware | logical tokens/s: 67782.5 | valid tokens/s: 67782.5 | physical tokens/s: 68987.2 | loss tokens/s: 49644.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.581961E-01 | lm-global-supervised-tokens: 1.228780E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.945866E-02 | lm-token-buffer-mib: 2.380613E+02 | global_load_balancing_loss: 9.996662E-01 | loss scale: 1.0 | grad norm: 0.329 | params norm: 11141.691 | params rms: 0.060329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:20:15.493872 | finish at 2026-10-17 19:51:19 + [2026-10-17 17:34:45] iteration 2351/ 2384 | consumed samples: 75232 | elapsed time per iteration (ms): 222309.2 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75467.8 | valid tokens/s: 75467.8 | physical tokens/s: 76086.3 | loss tokens/s: 57090.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.769996E-01 | lm-global-supervised-tokens: 1.269182E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.502994E-02 | lm-token-buffer-mib: 2.330350E+02 | global_load_balancing_loss: 9.986107E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11141.669 | params rms: 0.060329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:02:16.205175 | finish at 2026-10-17 19:37:02 + [2026-10-17 17:38:30] iteration 2352/ 2384 | consumed samples: 75264 | elapsed time per iteration (ms): 224720.7 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 74657.8 | valid tokens/s: 74657.8 | physical tokens/s: 75497.1 | loss tokens/s: 61809.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.920333E-01 | lm-global-supervised-tokens: 1.388992E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.409005E-02 | lm-token-buffer-mib: 2.358062E+02 | global_load_balancing_loss: 9.996703E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11141.645 | params rms: 0.060328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:59:51.061249 | finish at 2026-10-17 19:38:21 + [2026-10-17 17:42:09] iteration 2353/ 2384 | consumed samples: 75296 | elapsed time per iteration (ms): 218825.9 | throughput per GPU (TFLOP/s/GPU): 17.5 | MFU 5.61% | FLOPs estimator: workload-aware | logical tokens/s: 76669.2 | valid tokens/s: 76669.2 | physical tokens/s: 77524.7 | loss tokens/s: 59272.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.851427E-01 | lm-global-supervised-tokens: 1.297028E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.775635E-02 | lm-token-buffer-mib: 2.352625E+02 | global_load_balancing_loss: 9.986578E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11141.622 | params rms: 0.060328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:53:03.601875 | finish at 2026-10-17 19:35:13 + [2026-10-17 17:45:53] iteration 2354/ 2384 | consumed samples: 75328 | elapsed time per iteration (ms): 223597.6 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75032.8 | valid tokens/s: 75032.8 | physical tokens/s: 76319.9 | loss tokens/s: 59633.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.400755E-01 | lm-global-supervised-tokens: 1.333401E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.363826E-02 | lm-token-buffer-mib: 2.353372E+02 | global_load_balancing_loss: 1.000190E+00 | loss scale: 1.0 | grad norm: 0.292 | params norm: 11141.598 | params rms: 0.060328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:51:47.929387 | finish at 2026-10-17 19:37:41 + [2026-10-17 17:49:35] iteration 2355/ 2384 | consumed samples: 75360 | elapsed time per iteration (ms): 222359.4 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.02% | FLOPs estimator: workload-aware | logical tokens/s: 75450.8 | valid tokens/s: 75450.8 | physical tokens/s: 76458.6 | loss tokens/s: 55659.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.039707E-01 | lm-global-supervised-tokens: 1.237647E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.710891E-02 | lm-token-buffer-mib: 2.334335E+02 | global_load_balancing_loss: 1.000542E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11141.571 | params rms: 0.060328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:47:28.421398 | finish at 2026-10-17 19:37:03 + [2026-10-17 17:53:17] iteration 2356/ 2384 | consumed samples: 75392 | elapsed time per iteration (ms): 222072.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.28% | FLOPs estimator: workload-aware | logical tokens/s: 75548.4 | valid tokens/s: 75548.4 | physical tokens/s: 76516.5 | loss tokens/s: 59013.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.484653E-01 | lm-global-supervised-tokens: 1.310529E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.415170E-02 | lm-token-buffer-mib: 2.359273E+02 | global_load_balancing_loss: 1.000899E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11141.546 | params rms: 0.060328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:43:38.020634 | finish at 2026-10-17 19:36:55 + [2026-10-17 17:56:58] iteration 2357/ 2384 | consumed samples: 75424 | elapsed time per iteration (ms): 221236.8 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 75833.6 | valid tokens/s: 75833.6 | physical tokens/s: 76667.7 | loss tokens/s: 59592.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.289301E-01 | lm-global-supervised-tokens: 1.318401E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.146232E-02 | lm-token-buffer-mib: 2.342614E+02 | global_load_balancing_loss: 1.000700E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11141.521 | params rms: 0.060328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:39:33.394307 | finish at 2026-10-17 19:36:32 + [2026-10-17 18:00:41] iteration 2358/ 2384 | consumed samples: 75456 | elapsed time per iteration (ms): 222873.9 | throughput per GPU (TFLOP/s/GPU): 26.0 | MFU 8.32% | FLOPs estimator: workload-aware | logical tokens/s: 75276.5 | valid tokens/s: 75276.5 | physical tokens/s: 75509.9 | loss tokens/s: 52055.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.716033E-01 | lm-global-supervised-tokens: 1.160187E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.109636E-02 | lm-token-buffer-mib: 2.322952E+02 | global_load_balancing_loss: 1.001177E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11141.496 | params rms: 0.060328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:36:34.720735 | finish at 2026-10-17 19:37:16 + [2026-10-17 18:04:23] iteration 2359/ 2384 | consumed samples: 75488 | elapsed time per iteration (ms): 222207.0 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75502.5 | valid tokens/s: 75502.5 | physical tokens/s: 76343.7 | loss tokens/s: 60612.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.254078E-01 | lm-global-supervised-tokens: 1.346850E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.660935E-02 | lm-token-buffer-mib: 2.332932E+02 | global_load_balancing_loss: 1.001201E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11141.467 | params rms: 0.060327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:32:35.175203 | finish at 2026-10-17 19:36:59 + [2026-10-17 18:08:26] iteration 2360/ 2384 | consumed samples: 75520 | elapsed time per iteration (ms): 242211.7 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.19% | FLOPs estimator: workload-aware | logical tokens/s: 69266.7 | valid tokens/s: 69266.7 | physical tokens/s: 69669.2 | loss tokens/s: 52957.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.786213E-01 | lm-global-supervised-tokens: 1.282697E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.482358E-02 | lm-token-buffer-mib: 2.338764E+02 | global_load_balancing_loss: 1.000967E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11141.438 | params rms: 0.060327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:36:53.081491 | finish at 2026-10-17 19:45:19 +times across ranks (ms): + save-checkpoint: + rank 0: 24799.10 + rank 1: 24799.08 + rank 2: 24799.11 + rank 3: 24799.16 + rank 4: 24799.09 + rank 5: 24799.12 + rank 6: 24799.10 + rank 7: 24799.09 + rank 8: 24798.88 + rank 9: 24798.95 + rank 10: 24798.94 + rank 11: 24798.88 + rank 12: 24798.92 + rank 13: 24798.87 + rank 14: 24798.90 + rank 15: 24798.89 + rank 16: 24798.92 + rank 17: 24798.91 + rank 18: 24798.93 + rank 19: 24798.96 + rank 20: 24798.87 + rank 21: 24798.87 + rank 22: 24798.90 + rank 23: 24798.89 + rank 24: 24799.50 + rank 25: 24799.48 + rank 26: 24799.49 + rank 27: 24799.47 + rank 28: 24799.46 + rank 29: 24799.53 + rank 30: 24799.51 + rank 31: 24799.91 + rank 32: 24798.87 + rank 33: 24798.87 + rank 34: 24798.76 + rank 35: 24799.04 + rank 36: 24798.90 + rank 37: 24798.95 + rank 38: 24798.74 + rank 39: 24798.92 + rank 40: 24798.88 + rank 41: 24798.88 + rank 42: 24798.88 + rank 43: 24798.88 + rank 44: 24798.84 + rank 45: 24798.85 + rank 46: 24798.84 + rank 47: 24798.93 + rank 48: 24799.75 + rank 49: 24799.77 + rank 50: 24799.81 + rank 51: 24799.73 + rank 52: 24799.82 + rank 53: 24799.91 + rank 54: 24799.81 + rank 55: 24799.70 + rank 56: 24799.03 + rank 57: 24799.06 + rank 58: 24799.04 + rank 59: 24799.11 + rank 60: 24798.91 + rank 61: 24798.88 + rank 62: 24799.02 + rank 63: 24799.02 + [2026-10-17 18:12:35] iteration 2361/ 2384 | consumed samples: 75552 | elapsed time per iteration (ms): 224369.5 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.94% | FLOPs estimator: workload-aware | logical tokens/s: 74774.8 | valid tokens/s: 74774.7 | physical tokens/s: 75499.2 | loss tokens/s: 59929.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.361327E-01 | lm-global-supervised-tokens: 1.344639E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.125577E-01 | lm-token-buffer-mib: 2.350555E+02 | global_load_balancing_loss: 9.994248E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11141.410 | params rms: 0.060327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:26:00.497983 | finish at 2026-10-17 19:38:36 + [2026-10-17 18:16:22] iteration 2362/ 2384 | consumed samples: 75584 | elapsed time per iteration (ms): 226642.1 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 74025.1 | valid tokens/s: 74025.1 | physical tokens/s: 74935.6 | loss tokens/s: 55146.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.628219E-01 | lm-global-supervised-tokens: 1.249854E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.591265E-02 | lm-token-buffer-mib: 2.349855E+02 | global_load_balancing_loss: 9.998685E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11141.382 | params rms: 0.060327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:23:06.125473 | finish at 2026-10-17 19:39:28 + [2026-10-17 18:20:05] iteration 2363/ 2384 | consumed samples: 75616 | elapsed time per iteration (ms): 223370.2 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 75109.4 | valid tokens/s: 75109.4 | physical tokens/s: 76276.2 | loss tokens/s: 59199.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.447251E-01 | lm-global-supervised-tokens: 1.322334E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.799983E-02 | lm-token-buffer-mib: 2.375506E+02 | global_load_balancing_loss: 1.000381E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11141.353 | params rms: 0.060327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:18:10.774829 | finish at 2026-10-17 19:38:16 + [2026-10-17 18:23:47] iteration 2364/ 2384 | consumed samples: 75648 | elapsed time per iteration (ms): 222147.1 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 75522.8 | valid tokens/s: 75522.8 | physical tokens/s: 77116.5 | loss tokens/s: 57391.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.579518E-01 | lm-global-supervised-tokens: 1.274923E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.821753E-02 | lm-token-buffer-mib: 2.366209E+02 | global_load_balancing_loss: 1.000433E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11141.324 | params rms: 0.060327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:14:02.941971 | finish at 2026-10-17 19:37:50 + [2026-10-17 18:27:30] iteration 2365/ 2384 | consumed samples: 75680 | elapsed time per iteration (ms): 222410.3 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 75433.6 | valid tokens/s: 75433.6 | physical tokens/s: 76163.4 | loss tokens/s: 60535.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.279094E-01 | lm-global-supervised-tokens: 1.346378E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.219713E-02 | lm-token-buffer-mib: 2.345189E+02 | global_load_balancing_loss: 9.998090E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11141.294 | params rms: 0.060326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:10:25.795166 | finish at 2026-10-17 19:37:55 + [2026-10-17 18:31:13] iteration 2366/ 2384 | consumed samples: 75712 | elapsed time per iteration (ms): 223404.9 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.20% | FLOPs estimator: workload-aware | logical tokens/s: 75097.6 | valid tokens/s: 75097.6 | physical tokens/s: 75651.6 | loss tokens/s: 52939.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.932648E-01 | lm-global-supervised-tokens: 1.182696E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.262020E-02 | lm-token-buffer-mib: 2.344811E+02 | global_load_balancing_loss: 1.000150E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11141.263 | params rms: 0.060326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:07:01.288274 | finish at 2026-10-17 19:38:14 + [2026-10-17 18:34:57] iteration 2367/ 2384 | consumed samples: 75744 | elapsed time per iteration (ms): 223516.9 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 75060.0 | valid tokens/s: 75060.0 | physical tokens/s: 76288.0 | loss tokens/s: 55437.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.789155E-01 | lm-global-supervised-tokens: 1.239127E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.915189E-02 | lm-token-buffer-mib: 2.356863E+02 | global_load_balancing_loss: 1.001077E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11141.232 | params rms: 0.060326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:03:19.787605 | finish at 2026-10-17 19:38:16 + [2026-10-17 18:38:39] iteration 2368/ 2384 | consumed samples: 75776 | elapsed time per iteration (ms): 222327.3 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.25% | FLOPs estimator: workload-aware | logical tokens/s: 75461.8 | valid tokens/s: 75461.8 | physical tokens/s: 76326.8 | loss tokens/s: 53480.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.165713E-01 | lm-global-supervised-tokens: 1.189025E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.207300E-02 | lm-token-buffer-mib: 2.339821E+02 | global_load_balancing_loss: 1.000660E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11141.203 | params rms: 0.060326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:59:17.236294 | finish at 2026-10-17 19:37:56 + [2026-10-17 18:42:24] iteration 2369/ 2384 | consumed samples: 75808 | elapsed time per iteration (ms): 225540.6 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 74386.6 | valid tokens/s: 74386.6 | physical tokens/s: 75739.7 | loss tokens/s: 58015.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.288873E-01 | lm-global-supervised-tokens: 1.308495E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.767612E-02 | lm-token-buffer-mib: 2.349650E+02 | global_load_balancing_loss: 1.000248E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11141.174 | params rms: 0.060326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:56:23.108997 | finish at 2026-10-17 19:38:47 + [2026-10-17 18:46:28] iteration 2370/ 2384 | consumed samples: 75840 | elapsed time per iteration (ms): 243948.0 | throughput per GPU (TFLOP/s/GPU): 18.1 | MFU 5.79% | FLOPs estimator: workload-aware | logical tokens/s: 68773.7 | valid tokens/s: 68773.7 | physical tokens/s: 69390.3 | loss tokens/s: 51363.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.663929E-01 | lm-global-supervised-tokens: 1.252990E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.379879E-02 | lm-token-buffer-mib: 2.343483E+02 | global_load_balancing_loss: 1.000605E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11141.144 | params rms: 0.060326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:56:55.272430 | finish at 2026-10-17 19:43:24 + [2026-10-17 18:50:12] iteration 2371/ 2384 | consumed samples: 75872 | elapsed time per iteration (ms): 223477.1 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 75073.3 | valid tokens/s: 75073.3 | physical tokens/s: 75656.6 | loss tokens/s: 58456.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.204813E-01 | lm-global-supervised-tokens: 1.306358E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.102389E-02 | lm-token-buffer-mib: 2.343282E+02 | global_load_balancing_loss: 1.000207E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11141.112 | params rms: 0.060325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:48:25.201883 | finish at 2026-10-17 19:38:37 + [2026-10-17 18:53:52] iteration 2372/ 2384 | consumed samples: 75904 | elapsed time per iteration (ms): 220059.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 76239.5 | valid tokens/s: 76239.5 | physical tokens/s: 76630.5 | loss tokens/s: 55175.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.562590E-01 | lm-global-supervised-tokens: 1.214197E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.286157E-02 | lm-token-buffer-mib: 2.326078E+02 | global_load_balancing_loss: 1.000329E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11141.080 | params rms: 0.060325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:44:00.709622 | finish at 2026-10-17 19:37:53 + [2026-10-17 18:57:40] iteration 2373/ 2384 | consumed samples: 75936 | elapsed time per iteration (ms): 228220.5 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.47% | FLOPs estimator: workload-aware | logical tokens/s: 73513.1 | valid tokens/s: 73513.1 | physical tokens/s: 74700.3 | loss tokens/s: 51304.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.399871E-01 | lm-global-supervised-tokens: 1.170872E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.655370E-02 | lm-token-buffer-mib: 2.360803E+02 | global_load_balancing_loss: 9.999638E-01 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11141.048 | params rms: 0.060325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:41:50.425413 | finish at 2026-10-17 19:39:31 + [2026-10-17 19:01:24] iteration 2374/ 2384 | consumed samples: 75968 | elapsed time per iteration (ms): 224377.6 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 74772.2 | valid tokens/s: 74772.2 | physical tokens/s: 75720.4 | loss tokens/s: 55146.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.744265E-01 | lm-global-supervised-tokens: 1.237364E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.214943E-02 | lm-token-buffer-mib: 2.348498E+02 | global_load_balancing_loss: 9.996844E-01 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11141.017 | params rms: 0.060325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:37:23.776159 | finish at 2026-10-17 19:38:48 + [2026-10-17 19:05:07] iteration 2375/ 2384 | consumed samples: 76000 | elapsed time per iteration (ms): 222736.5 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.61% | FLOPs estimator: workload-aware | logical tokens/s: 75322.9 | valid tokens/s: 75322.9 | physical tokens/s: 75968.2 | loss tokens/s: 57156.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.135043E-01 | lm-global-supervised-tokens: 1.273082E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.584179E-02 | lm-token-buffer-mib: 2.351692E+02 | global_load_balancing_loss: 9.988273E-01 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11140.992 | params rms: 0.060325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:33:24.628676 | finish at 2026-10-17 19:38:32 + [2026-10-17 19:08:50] iteration 2376/ 2384 | consumed samples: 76032 | elapsed time per iteration (ms): 223176.7 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 75174.4 | valid tokens/s: 75174.4 | physical tokens/s: 76184.3 | loss tokens/s: 58724.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.124508E-01 | lm-global-supervised-tokens: 1.310589E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.380542E-02 | lm-token-buffer-mib: 2.348383E+02 | global_load_balancing_loss: 1.000534E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11140.970 | params rms: 0.060325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:29:45.413282 | finish at 2026-10-17 19:38:36 + [2026-10-17 19:12:31] iteration 2377/ 2384 | consumed samples: 76064 | elapsed time per iteration (ms): 220286.1 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 76160.8 | valid tokens/s: 76160.8 | physical tokens/s: 76841.7 | loss tokens/s: 58416.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.645155E-01 | lm-global-supervised-tokens: 1.286830E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.287969E-02 | lm-token-buffer-mib: 2.333653E+02 | global_load_balancing_loss: 1.001015E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11140.945 | params rms: 0.060325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:25:42.002916 | finish at 2026-10-17 19:38:13 + [2026-10-17 19:16:12] iteration 2378/ 2384 | consumed samples: 76096 | elapsed time per iteration (ms): 221189.0 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.19% | FLOPs estimator: workload-aware | logical tokens/s: 75850.0 | valid tokens/s: 75850.0 | physical tokens/s: 76656.0 | loss tokens/s: 57020.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.077727E-01 | lm-global-supervised-tokens: 1.261239E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.401456E-02 | lm-token-buffer-mib: 2.346887E+02 | global_load_balancing_loss: 1.000139E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11140.918 | params rms: 0.060324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:22:07.134170 | finish at 2026-10-17 19:38:19 + [2026-10-17 19:19:56] iteration 2379/ 2384 | consumed samples: 76128 | elapsed time per iteration (ms): 223799.1 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 74965.3 | valid tokens/s: 74965.3 | physical tokens/s: 75526.6 | loss tokens/s: 52820.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.932884E-01 | lm-global-supervised-tokens: 1.182107E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.224131E-02 | lm-token-buffer-mib: 2.345855E+02 | global_load_balancing_loss: 1.001727E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11140.891 | params rms: 0.060324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:18:38.995647 | finish at 2026-10-17 19:38:35 + [2026-10-17 19:23:59] iteration 2380/ 2384 | consumed samples: 76160 | elapsed time per iteration (ms): 243583.1 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 6.01% | FLOPs estimator: workload-aware | logical tokens/s: 68876.7 | valid tokens/s: 68876.7 | physical tokens/s: 69507.6 | loss tokens/s: 49424.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.449102E-01 | lm-global-supervised-tokens: 1.203897E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.551747E-02 | lm-token-buffer-mib: 2.331971E+02 | global_load_balancing_loss: 1.001267E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11140.864 | params rms: 0.060324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:16:14.332549 | finish at 2026-10-17 19:40:14 + [2026-10-17 19:27:41] iteration 2381/ 2384 | consumed samples: 76192 | elapsed time per iteration (ms): 221263.9 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 75824.3 | valid tokens/s: 75824.3 | physical tokens/s: 76587.8 | loss tokens/s: 53987.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.975965E-01 | lm-global-supervised-tokens: 1.194543E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.471700E-02 | lm-token-buffer-mib: 2.330941E+02 | global_load_balancing_loss: 9.996806E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11140.835 | params rms: 0.060324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:11:03.791812 | finish at 2026-10-17 19:38:44 + [2026-10-17 19:31:23] iteration 2382/ 2384 | consumed samples: 76224 | elapsed time per iteration (ms): 222421.5 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.20% | FLOPs estimator: workload-aware | logical tokens/s: 75429.8 | valid tokens/s: 75429.8 | physical tokens/s: 76188.9 | loss tokens/s: 59967.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.787159E-01 | lm-global-supervised-tokens: 1.333811E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.016818E-02 | lm-token-buffer-mib: 2.331569E+02 | global_load_balancing_loss: 9.991398E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11140.802 | params rms: 0.060324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:07:24.842957 | finish at 2026-10-17 19:38:48 + [2026-10-17 19:35:08] iteration 2383/ 2384 | consumed samples: 76256 | elapsed time per iteration (ms): 225526.5 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.98% | FLOPs estimator: workload-aware | logical tokens/s: 74391.3 | valid tokens/s: 74391.3 | physical tokens/s: 75295.2 | loss tokens/s: 56502.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.765473E-01 | lm-global-supervised-tokens: 1.274286E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.795324E-02 | lm-token-buffer-mib: 2.345378E+02 | global_load_balancing_loss: 9.988429E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11140.771 | params rms: 0.060324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:03:45.526459 | finish at 2026-10-17 19:38:54 + [2026-10-17 19:38:51] iteration 2384/ 2384 | consumed samples: 76288 | elapsed time per iteration (ms): 222158.7 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 75518.9 | valid tokens/s: 75518.9 | physical tokens/s: 76604.9 | loss tokens/s: 58222.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.096337E-01 | lm-global-supervised-tokens: 1.293464E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.254910E-02 | lm-token-buffer-mib: 2.354543E+02 | global_load_balancing_loss: 9.990426E-01 | loss scale: 1.0 | grad norm: 0.249 | params norm: 11140.739 | params rms: 0.060323 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:00:00 | finish at 2026-10-17 19:38:51 +times across ranks (ms): + evaluate: + rank 0: 630812.31 + rank 1: 630812.31 + rank 2: 630812.25 + rank 3: 630812.25 + rank 4: 630812.25 + rank 5: 630812.25 + rank 6: 630812.25 + rank 7: 630812.31 + rank 8: 630825.69 + rank 9: 630825.69 + rank 10: 630825.69 + rank 11: 630825.69 + rank 12: 630825.69 + rank 13: 630825.69 + rank 14: 630825.69 + rank 15: 630825.75 + rank 16: 630810.38 + rank 17: 630810.44 + rank 18: 630810.44 + rank 19: 630810.38 + rank 20: 630810.44 + rank 21: 630810.44 + rank 22: 630810.44 + rank 23: 630810.44 + rank 24: 630800.12 + rank 25: 630800.12 + rank 26: 630800.12 + rank 27: 630800.12 + rank 28: 630800.12 + rank 29: 630800.19 + rank 30: 630800.12 + rank 31: 630800.12 + rank 32: 630804.38 + rank 33: 630804.38 + rank 34: 630804.31 + rank 35: 630804.38 + rank 36: 630805.31 + rank 37: 630804.25 + rank 38: 630811.69 + rank 39: 630804.44 + rank 40: 630802.38 + rank 41: 630802.38 + rank 42: 630802.38 + rank 43: 630802.31 + rank 44: 630802.38 + rank 45: 630802.38 + rank 46: 630808.56 + rank 47: 630802.38 + rank 48: 630806.44 + rank 49: 630806.44 + rank 50: 630806.44 + rank 51: 630806.50 + rank 52: 630806.50 + rank 53: 630806.50 + rank 54: 630813.88 + rank 55: 630806.50 + rank 56: 630819.12 + rank 57: 630824.44 + rank 58: 630824.38 + rank 59: 630819.12 + rank 60: 630824.06 + rank 61: 630819.06 + rank 62: 630821.19 + rank 63: 630819.12 +------------------------------------------------------------------------------------------------------------------ + validation loss at iteration 2384 on validation set | lm loss value: 1.067154E+00 | lm loss PPL: 2.907094E+00 | +------------------------------------------------------------------------------------------------------------------ +wandb: updating run metadata +wandb: updating run metadata; uploading output.log; uploading wandb-summary.json +wandb: uploading output.log; uploading wandb-summary.json +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: uploading output.log; uploading wandb-summary.json +wandb: uploading history steps 2343-2343, summary, console lines 14212-14280 +wandb: +wandb: Run history: +wandb: batch-size ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: global_load_balancing_loss ▃▅▇▁▄▆▇▆▄▆▂▇▃▅▆▄▅█▇▆██▅▂▁█▆▃▄▆▄▅▅▅▃▄▁▄▅▆ +wandb: grad-absmax/attention::linear_qkv █▂▂▃▂▃▂▇▃▂▂▄▆▂▁▂▃▃▁▂▂▃▂▅▂▂▃▃▂▂▂▃▂▂▂▁▂▂▃▃ +wandb: grad-absmax/attention::o_proj ▂▃▁▃█▅▂▁▁▂▁▂▁▁▂▂▂▂▂▅▅▂▂▁▁▂▁▁▂▂▂▂▃▂▁▂▃▂▃▁ +wandb: grad-absmax/embedding ▂▆▃▃▃▃▃▁▁▂▄█▂▁▂▁▃▁▁▂▂▃▁▁▁▇▁▁▄▁▁▂▁▃▁▅▁▂▃▂ +wandb: grad-absmax/gated_delta::A_log ▁▁▂▂▅▂▂▁▆▂█▂▃█▄▆▃▁▃▄▁▁▂▁▁▃▇▂▅▂▄▂▃▄▁▁▃▂▃▂ +wandb: grad-absmax/gated_delta::conv1d ▁▃▆▁▅▂▁▁▂▂▄▃▅▇▃▃▂▂▂▃▁█▅▁▂▅▄▆▃▅▂▂▂▂▃▁▂▄▂▁ +wandb: grad-absmax/gated_delta::dt_bias ▄▂▂▁▃▁▂▆▂▂▁█▂▂▁▅▂▂▃▂▃▁▂▃▅▂▁▁▂▂▂▂▃▁▄▁▁▂▂▂ +wandb: grad-absmax/gated_delta::out_norm ▁▁▁▁▄▃█▁▃▁▂▁▂▂▂▁▃▆▂▁▂▂▂▂▂▂▂▂▃▁▂▂▂▁▃▂▂▆▃▃ +wandb: grad-absmax/gated_delta::out_proj ▄▁▂▁▁▂▄▅▃▁▇▂▃▂▂▅▅▅▂▂▂▃█▃▅▂▃▁▂▁▂▁▆▃▂▂▇▃▃▄ +wandb: +7,594 ... +wandb: +wandb: Run summary: +wandb: batch-size 32 +wandb: global_load_balancing_loss 0.99904 +wandb: grad-absmax/attention::linear_qkv 0.00114 +wandb: grad-absmax/attention::o_proj 6e-05 +wandb: grad-absmax/embedding 1e-05 +wandb: grad-absmax/gated_delta::A_log 0.00058 +wandb: grad-absmax/gated_delta::conv1d 0.02649 +wandb: grad-absmax/gated_delta::dt_bias 0.00057 +wandb: grad-absmax/gated_delta::out_norm 0.00045 +wandb: grad-absmax/gated_delta::out_proj 0.00027 +wandb: +7,594 ... +wandb: +wandb: 🚀 View run stage15_115B_8node_gdnlralpha_off_20261003a at: https://wandb.ai/huyiwen/qwen2.5_3b_mom_cpt/runs/acr8mx9p +wandb: ⭐️ View project at: https://wandb.ai/huyiwen/qwen2.5_3b_mom_cpt +wandb: Synced 5 W&B file(s), 0 media file(s), 108 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/wandb/run-20261011_160551-acr8mx9p/logs +[rank56]:[W1017 19:49:59.431075539 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node7.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node7.sh new file mode 100644 index 0000000000000000000000000000000000000000..93c63a5f057c1beff0fbd0caf2659b254a1d6879 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/resume_node7.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Stage16 launch: 512K seq length, loads stage15 final weights (resets iteration to 0). +# GDN LR alpha multiplier OFF: --gdn-lr-mult-alpha removed (default 1.0 = no per-proj LR scaling). +# Batch tokens unchanged: 128 docs x 131072 = 32 docs x 524288 = 16,777,216 tokens/step. +# TODO before launch: fill in --data-path (final link dir), --data-cache-path, +# --train-samples / --lr-decay-samples (budget_tokens / 524288, round to multiple of 32). +# NOTE: dataset not ready yet (raw .zst still downloading); DO NOT launch with TODO values. +# Command identical to stage15 except: 512K seq, gbs 32, gdn-lr-mult-alpha off, data, load+continual-pretrain, master-port 35873. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 7 --master-addr 10.239.252.55 --master-port 35873 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 524288 --max-position-embeddings 524288 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_{0..74}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 32 --lr 5e-4 --train-samples 76288 --lr-warmup-samples 0 --lr-decay-samples 76288 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 32 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type p2p --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 32 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb --wandb-exp-name stage15_115B_8node_gdnlralpha_off_20261003a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit